From 635d25c74cc98a0082d8886e7efa2c4d1a95d7d6 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Fri, 10 Jul 2026 12:51:19 +0200 Subject: [PATCH] chore: consolidate platform split checks --- .gitea/workflows/dependency-audit.yml | 34 + .gitea/workflows/module-matrix.yml | 31 + .gitignore | 12 + README.md | 41 +- alembic/env.py | 1 + ...70_drop_mail_credential_override_policy.py | 4 +- ...1f8d4c2a0b7e_editable_campaign_versions.py | 2 +- .../2c3d4e5f6a7b_auth_sessions_and_rbac.py | 98 +- .../2e3f4a5b6c7d_namespace_platform_tables.py | 78 + .../3d4e5f6a7b8c_file_storage_backend.py | 20 +- .../3f4a5b6c7d8e_core_change_sequence.py | 111 + alembic/versions/4e5f6a7b8c9d_file_folders.py | 8 +- ...f6a7b8c9d0e_campaign_version_user_locks.py | 2 +- ...8c9d0e1f2a3b_accounts_tenant_admin_rbac.py | 100 +- ...d0e1f2a3b4c_system_governance_templates.py | 71 +- ...2d3e4f5_permission_catalogue_refinement.py | 18 +- ...b57c5b216bce_initial_persistence_models.py | 82 +- ...4f5a6b7c8_mail_profiles_and_cookie_csrf.py | 18 +- ...5a6b7c8d9_mail_profile_policy_hierarchy.py | 4 +- .../f5a6b7c8d9e0_hierarchical_settings.py | 4 +- dev/postgres/.env.example | 5 + dev/postgres/README.md | 126 + dev/postgres/docker-compose.yml | 22 + dev/production-like/.env.example | 9 + dev/production-like/README.md | 49 + dev/production-like/docker-compose.yml | 37 + docs/ACCESS_EXTRACTION_PLAN.md | 456 --- docs/ACCESS_RBAC_MODEL.md | 288 ++ docs/ACTION_EFFECT_AUTOMATION_LAYER.md | 121 + docs/CATALOG_TRUST_AND_LICENSING.md | 185 - docs/CODEX_WORKFLOW.md | 6 + docs/CONFIGURATION_PACKAGES.md | 48 + docs/DEPENDENCY_AUDITS.md | 67 + docs/DEPLOYMENT_OPERATOR_GUIDE.md | 390 +++ docs/DOCUMENTATION_MAP.md | 51 + docs/EVENTS_AND_AUDIT.md | 180 + docs/GITEA_ISSUES.md | 35 +- ...RNANCE_MANIFEST.md => GOVERNANCE_MODEL.md} | 145 +- docs/GOVOPLAN_MASTER_ROADMAP.md | 663 ++++ docs/GOVOPLAN_MODULE_ROADMAP.md | 79 - docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md | 102 + docs/MODULE_ARCHITECTURE.md | 516 ++- docs/POLICY_CONTRACTS.md | 105 + docs/POSTBOX_E2EE_ARCHITECTURE.md | 134 + docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md | 276 ++ docs/RBAC_MANIFEST.md | 291 -- docs/RELEASE_CATALOG_WORKFLOW.md | 129 - docs/RELEASE_DEPENDENCIES.md | 822 +++-- docs/REMOTE_WEBUI_BUNDLES.md | 161 + docs/UI_UX_DECISION_LEDGER.md | 216 ++ docs/audits/2026-07-09-dependency-audit.md | 71 + docs/gitea-labels.json | 18 + docs/migration-release-baselines.json | 4 + docs/module-package-catalog.example.json | 61 + pyproject.toml | 6 +- requirements-dev.txt | 7 + requirements-release.txt | 23 +- scripts/check-dependency-audits.sh | 37 + scripts/check-dependency-hygiene.sh | 158 + scripts/check-focused.sh | 12 +- scripts/check-module-matrix.sh | 23 + scripts/check_dependency_boundaries.py | 10 +- scripts/generate-release-catalog.py | 43 + scripts/gitea-backlog-import.py | 10 +- scripts/gitea-import-all-backlogs.py | 9 +- scripts/gitea-migrate-org-labels.py | 250 ++ scripts/gitea-sync-labels.py | 49 +- scripts/gitea-sync-wiki.py | 58 +- scripts/gitea-todo-import.py | 5 +- scripts/gitea_common.py | 41 + scripts/launch-dev.sh | 16 + scripts/launch-production-like-dev.sh | 238 ++ scripts/module-installer-rollback-drill.py | 512 +++ scripts/postgres-integration-check.py | 128 + scripts/push-release-tag.sh | 66 +- scripts/release-migration-audit.py | 422 +++ src/govoplan_core/admin/models.py | 2 +- src/govoplan_core/api/v1/schemas.py | 65 + src/govoplan_core/audit/logging.py | 137 +- .../commands/module_installer.py | 79 +- src/govoplan_core/core/access.py | 249 +- src/govoplan_core/core/change_sequence.py | 253 ++ .../core/configuration_control.py | 418 +++ .../core/configuration_packages.py | 883 +++++ .../core/configuration_safety.py | 395 +++ src/govoplan_core/core/events.py | 86 +- src/govoplan_core/core/identity.py | 46 + src/govoplan_core/core/lifecycle.py | 2 + src/govoplan_core/core/module_installer.py | 202 +- src/govoplan_core/core/module_license.py | 148 +- src/govoplan_core/core/module_management.py | 15 +- .../core/module_package_catalog.py | 95 +- src/govoplan_core/core/modules.py | 61 + src/govoplan_core/core/organizations.py | 85 + src/govoplan_core/core/pagination.py | 70 + src/govoplan_core/core/policy.py | 124 + src/govoplan_core/core/registry.py | 75 + src/govoplan_core/db/bootstrap.py | 1 + src/govoplan_core/db/migrations.py | 161 +- src/govoplan_core/db/session.py | 15 +- src/govoplan_core/devserver.py | 21 +- src/govoplan_core/i18n.py | 152 + src/govoplan_core/privacy/retention.py | 15 +- src/govoplan_core/security/secrets.py | 16 + src/govoplan_core/server/app.py | 6 +- .../server/conditional_requests.py | 123 + src/govoplan_core/server/default_config.py | 13 +- src/govoplan_core/server/fastapi.py | 18 +- src/govoplan_core/server/platform.py | 6 + src/govoplan_core/server/route_validation.py | 73 + src/govoplan_core/settings.py | 4 +- tests/db_isolation.py | 26 + tests/test_access_contracts.py | 509 ++- tests/test_api_smoke.py | 3083 ++++++++++++++++- tests/test_change_sequence.py | 121 + tests/test_conditional_requests.py | 103 + tests/test_core_events.py | 116 + tests/test_database_migrations.py | 128 +- tests/test_devserver.py | 5 + tests/test_identity_organization_contracts.py | 186 + tests/test_module_system.py | 867 ++++- tests/test_pagination.py | 33 + tests/test_policy_contracts.py | 125 + tests/test_release_migration_audit.py | 141 + webui/index.html | 7 + webui/package-lock.json | 93 +- webui/package.json | 12 +- webui/package.release.json | 5 +- webui/public/app-icon.svg | 11 + webui/public/apple-touch-icon.png | Bin 0 -> 7866 bytes webui/public/favicon-16x16.png | Bin 0 -> 663 bytes webui/public/favicon-32x32.png | Bin 0 -> 1278 bytes webui/public/favicon-48x48.png | Bin 0 -> 1971 bytes webui/public/favicon-96x96.png | Bin 0 -> 4016 bytes webui/public/favicon.ico | Bin 0 -> 3966 bytes webui/public/favicon.svg | 10 + webui/public/maskable-icon.svg | 11 + webui/public/safari-pinned-tab.svg | 3 + webui/public/site.webmanifest | 30 + webui/public/web-app-manifest-192x192.png | Bin 0 -> 8386 bytes .../web-app-manifest-512x512-maskable.png | Bin 0 -> 15396 bytes webui/public/web-app-manifest-512x512.png | Bin 0 -> 24080 bytes webui/scripts/audit-i18n-structural.mjs | 347 ++ webui/scripts/test-module-permutations.mjs | 24 +- webui/src/App.tsx | 329 +- webui/src/api/admin.ts | 256 +- webui/src/api/auth.ts | 10 +- webui/src/api/client.ts | 181 +- webui/src/api/platform.ts | 9 + webui/src/components/AccessBoundary.tsx | 30 +- webui/src/components/ActionBlockerHint.tsx | 64 + webui/src/components/AdvancedOptionsPanel.tsx | 33 + webui/src/components/Card.tsx | 47 +- webui/src/components/ColorPickerField.tsx | 105 + webui/src/components/ConfirmDialog.tsx | 24 +- webui/src/components/ConnectionTree.tsx | 84 + webui/src/components/DateTimeField.tsx | 231 ++ webui/src/components/Dialog.tsx | 16 +- webui/src/components/DismissibleAlert.tsx | 20 +- webui/src/components/ExplorerTree.tsx | 109 +- webui/src/components/FileDropZone.tsx | 56 +- webui/src/components/FormField.tsx | 5 +- webui/src/components/GuidedConfigDialog.tsx | 53 + webui/src/components/GuidedReviewList.tsx | 42 + webui/src/components/LoadingFrame.tsx | 21 +- webui/src/components/LoadingIndicator.tsx | 14 +- webui/src/components/MessageDisplayPanel.tsx | 190 +- webui/src/components/PageTitle.tsx | 13 +- webui/src/components/PasswordField.tsx | 32 +- webui/src/components/PolicySourcePath.tsx | 58 +- webui/src/components/PolicyTable.tsx | 131 + webui/src/components/SegmentedControl.tsx | 76 + webui/src/components/StatusBadge.tsx | 5 +- webui/src/components/ToggleSwitch.tsx | 5 +- webui/src/components/UnsavedChangesGuard.tsx | 116 +- .../src/components/admin/AdminIconButton.tsx | 7 +- .../src/components/admin/AdminPageLayout.tsx | 14 +- .../components/admin/AdminSelectionList.tsx | 55 +- webui/src/components/admin/adminUtils.ts | 32 +- .../components/email/EmailAddressInput.tsx | 100 +- webui/src/components/help/InlineHelp.tsx | 68 +- .../mail/MailServerSettingsPanel.tsx | 281 +- webui/src/components/table/DataGrid.tsx | 821 ++--- webui/src/features/PlaceholderPage.tsx | 12 +- webui/src/features/auth/LoginModal.tsx | 38 +- webui/src/features/auth/PublicLandingPage.tsx | 56 +- .../src/features/dashboard/DashboardPage.tsx | 143 +- .../privacy/RetentionPolicyManagement.tsx | 357 +- webui/src/features/privacy/policyLogic.ts | 29 + webui/src/features/settings/SettingsPage.tsx | 460 ++- webui/src/i18n/LanguageContext.tsx | 382 ++ webui/src/i18n/generatedTranslations.ts | 1118 ++++++ webui/src/index.ts | 23 +- webui/src/layout/AppShell.tsx | 8 +- webui/src/layout/BreadcrumbBar.tsx | 78 +- webui/src/layout/HelpMenu.tsx | 93 +- webui/src/layout/IconRail.tsx | 71 +- webui/src/layout/LanguageMenu.tsx | 48 + webui/src/layout/ModuleSubnav.tsx | 55 +- webui/src/layout/Titlebar.tsx | 141 +- webui/src/platform/modules.ts | 70 +- webui/src/styles/components.css | 856 ++++- webui/src/styles/forms.css | 28 + webui/src/styles/layout.css | 80 +- webui/src/styles/retention-policies.css | 12 - webui/src/styles/tables.css | 144 + webui/src/styles/tokens.css | 41 + webui/src/types.ts | 141 +- webui/src/utils/delta.ts | 32 + webui/src/utils/deltaHooks.ts | 27 + webui/src/utils/fieldHelp.ts | 142 +- webui/src/utils/helpContext.ts | 62 +- webui/tests/mail-components.test.tsx | 32 +- webui/tests/privacy-policy.test.ts | 34 + webui/tsconfig.module-tests.json | 2 + webui/vite.config.ts | 15 +- 216 files changed, 23336 insertions(+), 4077 deletions(-) create mode 100644 .gitea/workflows/dependency-audit.yml create mode 100644 .gitea/workflows/module-matrix.yml create mode 100644 alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py create mode 100644 alembic/versions/3f4a5b6c7d8e_core_change_sequence.py create mode 100644 dev/postgres/.env.example create mode 100644 dev/postgres/README.md create mode 100644 dev/postgres/docker-compose.yml create mode 100644 dev/production-like/.env.example create mode 100644 dev/production-like/README.md create mode 100644 dev/production-like/docker-compose.yml delete mode 100644 docs/ACCESS_EXTRACTION_PLAN.md create mode 100644 docs/ACCESS_RBAC_MODEL.md create mode 100644 docs/ACTION_EFFECT_AUTOMATION_LAYER.md delete mode 100644 docs/CATALOG_TRUST_AND_LICENSING.md create mode 100644 docs/DEPENDENCY_AUDITS.md create mode 100644 docs/DEPLOYMENT_OPERATOR_GUIDE.md create mode 100644 docs/DOCUMENTATION_MAP.md create mode 100644 docs/EVENTS_AND_AUDIT.md rename docs/{SYSTEM_GOVERNANCE_MANIFEST.md => GOVERNANCE_MODEL.md} (52%) create mode 100644 docs/GOVOPLAN_MASTER_ROADMAP.md delete mode 100644 docs/GOVOPLAN_MODULE_ROADMAP.md create mode 100644 docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md create mode 100644 docs/POLICY_CONTRACTS.md create mode 100644 docs/POSTBOX_E2EE_ARCHITECTURE.md create mode 100644 docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md delete mode 100644 docs/RBAC_MANIFEST.md delete mode 100644 docs/RELEASE_CATALOG_WORKFLOW.md create mode 100644 docs/REMOTE_WEBUI_BUNDLES.md create mode 100644 docs/UI_UX_DECISION_LEDGER.md create mode 100644 docs/audits/2026-07-09-dependency-audit.md create mode 100644 docs/migration-release-baselines.json create mode 100644 scripts/check-dependency-audits.sh create mode 100644 scripts/check-dependency-hygiene.sh create mode 100644 scripts/check-module-matrix.sh create mode 100644 scripts/gitea-migrate-org-labels.py create mode 100644 scripts/launch-production-like-dev.sh create mode 100644 scripts/module-installer-rollback-drill.py create mode 100644 scripts/postgres-integration-check.py create mode 100644 scripts/release-migration-audit.py create mode 100644 src/govoplan_core/core/change_sequence.py create mode 100644 src/govoplan_core/core/configuration_control.py create mode 100644 src/govoplan_core/core/configuration_packages.py create mode 100644 src/govoplan_core/core/configuration_safety.py create mode 100644 src/govoplan_core/core/identity.py create mode 100644 src/govoplan_core/core/organizations.py create mode 100644 src/govoplan_core/core/pagination.py create mode 100644 src/govoplan_core/core/policy.py create mode 100644 src/govoplan_core/i18n.py create mode 100644 src/govoplan_core/server/conditional_requests.py create mode 100644 src/govoplan_core/server/route_validation.py create mode 100644 tests/db_isolation.py create mode 100644 tests/test_change_sequence.py create mode 100644 tests/test_conditional_requests.py create mode 100644 tests/test_core_events.py create mode 100644 tests/test_identity_organization_contracts.py create mode 100644 tests/test_pagination.py create mode 100644 tests/test_policy_contracts.py create mode 100644 tests/test_release_migration_audit.py create mode 100644 webui/public/app-icon.svg create mode 100644 webui/public/apple-touch-icon.png create mode 100644 webui/public/favicon-16x16.png create mode 100644 webui/public/favicon-32x32.png create mode 100644 webui/public/favicon-48x48.png create mode 100644 webui/public/favicon-96x96.png create mode 100644 webui/public/favicon.ico create mode 100644 webui/public/favicon.svg create mode 100644 webui/public/maskable-icon.svg create mode 100644 webui/public/safari-pinned-tab.svg create mode 100644 webui/public/site.webmanifest create mode 100644 webui/public/web-app-manifest-192x192.png create mode 100644 webui/public/web-app-manifest-512x512-maskable.png create mode 100644 webui/public/web-app-manifest-512x512.png create mode 100644 webui/scripts/audit-i18n-structural.mjs create mode 100644 webui/src/components/ActionBlockerHint.tsx create mode 100644 webui/src/components/AdvancedOptionsPanel.tsx create mode 100644 webui/src/components/ColorPickerField.tsx create mode 100644 webui/src/components/ConnectionTree.tsx create mode 100644 webui/src/components/DateTimeField.tsx create mode 100644 webui/src/components/GuidedConfigDialog.tsx create mode 100644 webui/src/components/GuidedReviewList.tsx create mode 100644 webui/src/components/PolicyTable.tsx create mode 100644 webui/src/components/SegmentedControl.tsx create mode 100644 webui/src/features/privacy/policyLogic.ts create mode 100644 webui/src/i18n/LanguageContext.tsx create mode 100644 webui/src/i18n/generatedTranslations.ts create mode 100644 webui/src/layout/LanguageMenu.tsx create mode 100644 webui/src/utils/delta.ts create mode 100644 webui/src/utils/deltaHooks.ts create mode 100644 webui/tests/privacy-policy.test.ts diff --git a/.gitea/workflows/dependency-audit.yml b/.gitea/workflows/dependency-audit.yml new file mode 100644 index 0000000..794e769 --- /dev/null +++ b/.gitea/workflows/dependency-audit.yml @@ -0,0 +1,34 @@ +name: Dependency Audit + +on: + pull_request: + push: + branches: + - main + schedule: + - cron: "23 3 * * 1" + +jobs: + dependency-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install backend dev audit dependencies + run: | + python -m venv .venv + .venv/bin/python -m pip install --upgrade pip + .venv/bin/python -m pip install -r requirements-release.txt + .venv/bin/python -m pip install 'pip-audit>=2.9,<3' + - name: Install WebUI release dependencies + working-directory: webui + run: | + node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); pkg.dependencies=rel.dependencies; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');" + npm install + - name: Run dependency audits + run: bash scripts/check-dependency-audits.sh diff --git a/.gitea/workflows/module-matrix.yml b/.gitea/workflows/module-matrix.yml new file mode 100644 index 0000000..194c941 --- /dev/null +++ b/.gitea/workflows/module-matrix.yml @@ -0,0 +1,31 @@ +name: Module Matrix + +on: + pull_request: + push: + branches: + - main + +jobs: + module-matrix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install backend release dependencies + run: | + python -m venv .venv + .venv/bin/python -m pip install --upgrade pip + .venv/bin/python -m pip install -r requirements-release.txt + - name: Install WebUI release dependencies with test scripts + working-directory: webui + run: | + node -e "const fs=require('fs'); const pkg=JSON.parse(fs.readFileSync('package.json')); const rel=JSON.parse(fs.readFileSync('package.release.json')); pkg.dependencies=rel.dependencies; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');" + npm install + - name: Run module matrix and contract tests + run: bash scripts/check-module-matrix.sh diff --git a/.gitignore b/.gitignore index 32e22c2..6750d45 100644 --- a/.gitignore +++ b/.gitignore @@ -136,6 +136,18 @@ dist .yarn/install-state.gz .pnp.* +# Local WebUI test/build scratch directories +.component-test-build/ +.module-test-build/ +.policy-test-build/ +.template-preview-test-build/ +.import-test-build/ +webui/.component-test-build/ +webui/.module-test-build/ +webui/.policy-test-build/ +webui/.template-preview-test-build/ +webui/.import-test-build/ + # ---> Python # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index d2fe4a9..2d5bab1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # govoplan-core -GovOPlaN core is the platform runner and shared foundation. It owns the server entry point, database/session primitives, tenant and RBAC infrastructure, governance policy, audit/auth helpers, module discovery, migration registration, and the shared WebUI shell. Feature code is supplied by installed modules. +GovOPlaN core is the platform runner and shared foundation. It owns the server entry point, database/session primitives, module discovery, migration orchestration, capability contracts, install/uninstall orchestration, and the shared WebUI shell. Platform and feature behavior is supplied by installed modules. ## Repository ownership @@ -9,22 +9,28 @@ Core owns: - `govoplan_core.server.app:app`, the FastAPI entry point used by uvicorn - `GovoplanServerConfig`, module discovery, registry validation, and route aggregation - SQLAlchemy base/session helpers and module migration registration -- tenant/account/session/RBAC/governance/audit models and services -- core API routes for auth, admin, platform metadata, audit, and system health +- kernel APIs for platform metadata, module lifecycle, health, and development diagnostics - `@govoplan/core-webui`, including login, CSRF/API helpers, shell layout, generic UI components, IconRail, DataGrid, access boundaries, and module route/nav contracts -Feature modules own their backend routers, models, migrations, permissions, frontend packages, nav items, and route contributions. Core should not import feature pages directly; it imports module manifests and renders their route contributions. +Platform and feature modules own their backend routers, models, migrations, +permissions, frontend packages, nav items, and route contributions. Access, +tenancy, policy, audit, and admin behavior live in their owning platform +modules. Core should not import feature pages directly; it imports module +manifests and renders their route contributions. ## Governance docs Canonical policy documents live in `docs/`: -- [RBAC_MANIFEST.md](docs/RBAC_MANIFEST.md) -- [SYSTEM_GOVERNANCE_MANIFEST.md](docs/SYSTEM_GOVERNANCE_MANIFEST.md) +- [DOCUMENTATION_MAP.md](docs/DOCUMENTATION_MAP.md) +- [ACCESS_RBAC_MODEL.md](docs/ACCESS_RBAC_MODEL.md) +- [GOVERNANCE_MODEL.md](docs/GOVERNANCE_MODEL.md) - [MODULE_ARCHITECTURE.md](docs/MODULE_ARCHITECTURE.md) +- [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md) - [CODEX_WORKFLOW.md](docs/CODEX_WORKFLOW.md) -Modules may define module-specific permissions and policy behavior, but the platform-level permission model and governance hierarchy belong here. +Modules define module-specific permissions and policy behavior. Shared DTOs and +composition rules live in core only where they are stable kernel contracts. ## Backend development @@ -35,7 +41,7 @@ cd /mnt/DATA/git/govoplan-core ./.venv/bin/python -m pip install -r requirements-dev.txt ``` -Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `tenancy,access,admin,policy,audit,campaigns,files,mail`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation. +Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation. ```bash cd /mnt/DATA/git/govoplan-core @@ -55,13 +61,24 @@ ENABLED_MODULES=access,campaigns ./.venv/bin/python -m govoplan_core.devserver \ The runner loads the same `GovoplanServerConfig` as `govoplan_core.server.app:app`, builds the platform registry, and passes core plus enabled module source roots to uvicorn as reload directories. After reinstalling the editable package, the same command is also available as `govoplan-devserver`. -The default development SQLite database lives at `runtime/multimailer-dev.db`, alongside other local runtime state. +The default development database is PostgreSQL at `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev`. Store the password in `~/.pgpass`. To force the disposable SQLite fallback, run with `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite`; that database lives below `runtime/`. Local devserver runs do not require Redis. `CELERY_ENABLED` defaults to `false`, so campaign queue actions update database state without publishing Celery tasks. Use the synchronous send flow for local send tests, or set `CELERY_ENABLED=true` only when a Redis broker and worker are running. -If the configured local SQLite database is missing or empty, `govoplan_core.devserver` enables the development bootstrap before loading settings. This creates the schema and the default development login on startup. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead. +To run the production-like local profile with PostgreSQL, Redis, a Celery +worker, explicit module configuration, and persistent local file storage: -To verify the effective runtime paths and missing-SQLite bootstrap without starting uvicorn, run the smoke mode: +```bash +cd /mnt/DATA/git/govoplan-core +scripts/launch-production-like-dev.sh +``` + +See [dev/production-like/README.md](dev/production-like/README.md) for ports, +environment overrides, and cleanup commands. + +`govoplan_core.devserver` enables the development bootstrap before loading settings. In dev, startup migrations create or upgrade the schema and the bootstrap creates the default development login if needed. Explicitly setting `DEV_BOOTSTRAP_ENABLED=false` disables this convenience. Production deployments should use migrations and managed database provisioning instead. + +To verify the effective runtime paths and bootstrap behavior without starting uvicorn, run the smoke mode: ```bash cd /mnt/DATA/git/govoplan-core @@ -72,6 +89,8 @@ The smoke mode prints the effective config, runtime root, database URL, modules, `requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md). +For the install/runtime configuration contract and operator deployment flow, see [DEPLOYMENT_OPERATOR_GUIDE.md](docs/DEPLOYMENT_OPERATOR_GUIDE.md). + ## WebUI development Install and run from the core WebUI host: diff --git a/alembic/env.py b/alembic/env.py index efe5273..59f667e 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -7,6 +7,7 @@ from sqlalchemy import engine_from_config, pool from govoplan_access.backend.db import models as access_models # noqa: F401 - populate access metadata from govoplan_core.admin import models as core_admin_models # noqa: F401 - populate core admin metadata +from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata from govoplan_core.core.migrations import migration_metadata_plan from govoplan_core.db.base import Base from govoplan_core.server.default_config import get_server_config diff --git a/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py b/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py index 8f8bc82..de2fe00 100644 --- a/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py +++ b/alembic/versions/1b2c3d4e5f70_drop_mail_credential_override_policy.py @@ -94,10 +94,10 @@ def _scrub_policy_column(table_name: str) -> None: def upgrade() -> None: inspector = sa.inspect(op.get_bind()) tables = set(inspector.get_table_names()) - for table_name in ("system_settings", "tenants"): + for table_name in ("core_system_settings", "tenancy_tenants"): if table_name in tables and "settings" in {column["name"] for column in inspector.get_columns(table_name)}: _scrub_settings_table(table_name) - for table_name in ("users", "groups", "campaigns"): + for table_name in ("access_users", "access_groups", "campaigns"): if table_name in tables and "mail_profile_policy" in {column["name"] for column in inspector.get_columns(table_name)}: _scrub_policy_column(table_name) diff --git a/alembic/versions/1f8d4c2a0b7e_editable_campaign_versions.py b/alembic/versions/1f8d4c2a0b7e_editable_campaign_versions.py index 33a085d..8f30209 100644 --- a/alembic/versions/1f8d4c2a0b7e_editable_campaign_versions.py +++ b/alembic/versions/1f8d4c2a0b7e_editable_campaign_versions.py @@ -30,7 +30,7 @@ def upgrade() -> None: batch_op.add_column(sa.Column("locked_by_user_id", sa.String(length=36), nullable=True)) batch_op.create_foreign_key( op.f("fk_campaign_versions_locked_by_user_id_users"), - "users", + "access_users", ["locked_by_user_id"], ["id"], ondelete="SET NULL", diff --git a/alembic/versions/2c3d4e5f6a7b_auth_sessions_and_rbac.py b/alembic/versions/2c3d4e5f6a7b_auth_sessions_and_rbac.py index d3756a9..5babb3f 100644 --- a/alembic/versions/2c3d4e5f6a7b_auth_sessions_and_rbac.py +++ b/alembic/versions/2c3d4e5f6a7b_auth_sessions_and_rbac.py @@ -17,67 +17,67 @@ depends_on = None def upgrade() -> None: - with op.batch_alter_table("users") as batch_op: + with op.batch_alter_table("access_users") as batch_op: batch_op.add_column(sa.Column("auth_provider", sa.String(length=50), nullable=False, server_default="local")) batch_op.add_column(sa.Column("password_hash", sa.String(length=500), nullable=True)) batch_op.add_column(sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True)) op.create_table( - "user_group_memberships", + "access_user_group_memberships", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("tenant_id", sa.String(length=36), nullable=False), sa.Column("user_id", sa.String(length=36), nullable=False), sa.Column("group_id", sa.String(length=36), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["group_id"], ["groups.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["group_id"], ["access_groups.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("tenant_id", "user_id", "group_id", name="uq_user_group_memberships"), ) - op.create_index(op.f("ix_user_group_memberships_tenant_id"), "user_group_memberships", ["tenant_id"]) - op.create_index(op.f("ix_user_group_memberships_user_id"), "user_group_memberships", ["user_id"]) - op.create_index(op.f("ix_user_group_memberships_group_id"), "user_group_memberships", ["group_id"]) + op.create_index(op.f("ix_access_user_group_memberships_tenant_id"), "access_user_group_memberships", ["tenant_id"]) + op.create_index(op.f("ix_access_user_group_memberships_user_id"), "access_user_group_memberships", ["user_id"]) + op.create_index(op.f("ix_access_user_group_memberships_group_id"), "access_user_group_memberships", ["group_id"]) op.create_table( - "user_role_assignments", + "access_user_role_assignments", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("tenant_id", sa.String(length=36), nullable=False), sa.Column("user_id", sa.String(length=36), nullable=False), sa.Column("role_id", sa.String(length=36), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("tenant_id", "user_id", "role_id", name="uq_user_role_assignments"), ) - op.create_index(op.f("ix_user_role_assignments_tenant_id"), "user_role_assignments", ["tenant_id"]) - op.create_index(op.f("ix_user_role_assignments_user_id"), "user_role_assignments", ["user_id"]) - op.create_index(op.f("ix_user_role_assignments_role_id"), "user_role_assignments", ["role_id"]) + op.create_index(op.f("ix_access_user_role_assignments_tenant_id"), "access_user_role_assignments", ["tenant_id"]) + op.create_index(op.f("ix_access_user_role_assignments_user_id"), "access_user_role_assignments", ["user_id"]) + op.create_index(op.f("ix_access_user_role_assignments_role_id"), "access_user_role_assignments", ["role_id"]) op.create_table( - "group_role_assignments", + "access_group_role_assignments", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("tenant_id", sa.String(length=36), nullable=False), sa.Column("group_id", sa.String(length=36), nullable=False), sa.Column("role_id", sa.String(length=36), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["group_id"], ["groups.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["group_id"], ["access_groups.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("tenant_id", "group_id", "role_id", name="uq_group_role_assignments"), ) - op.create_index(op.f("ix_group_role_assignments_tenant_id"), "group_role_assignments", ["tenant_id"]) - op.create_index(op.f("ix_group_role_assignments_group_id"), "group_role_assignments", ["group_id"]) - op.create_index(op.f("ix_group_role_assignments_role_id"), "group_role_assignments", ["role_id"]) + op.create_index(op.f("ix_access_group_role_assignments_tenant_id"), "access_group_role_assignments", ["tenant_id"]) + op.create_index(op.f("ix_access_group_role_assignments_group_id"), "access_group_role_assignments", ["group_id"]) + op.create_index(op.f("ix_access_group_role_assignments_role_id"), "access_group_role_assignments", ["role_id"]) op.create_table( - "auth_sessions", + "access_auth_sessions", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("tenant_id", sa.String(length=36), nullable=False), sa.Column("user_id", sa.String(length=36), nullable=False), @@ -89,42 +89,42 @@ def upgrade() -> None: sa.Column("ip_address", sa.String(length=100), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["user_id"], ["access_users.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("token_hash"), ) - op.create_index(op.f("ix_auth_sessions_tenant_id"), "auth_sessions", ["tenant_id"]) - op.create_index(op.f("ix_auth_sessions_user_id"), "auth_sessions", ["user_id"]) - op.create_index(op.f("ix_auth_sessions_token_hash"), "auth_sessions", ["token_hash"]) - op.create_index(op.f("ix_auth_sessions_expires_at"), "auth_sessions", ["expires_at"]) - op.create_index(op.f("ix_auth_sessions_revoked_at"), "auth_sessions", ["revoked_at"]) + op.create_index(op.f("ix_access_auth_sessions_tenant_id"), "access_auth_sessions", ["tenant_id"]) + op.create_index(op.f("ix_access_auth_sessions_user_id"), "access_auth_sessions", ["user_id"]) + op.create_index(op.f("ix_access_auth_sessions_token_hash"), "access_auth_sessions", ["token_hash"]) + op.create_index(op.f("ix_access_auth_sessions_expires_at"), "access_auth_sessions", ["expires_at"]) + op.create_index(op.f("ix_access_auth_sessions_revoked_at"), "access_auth_sessions", ["revoked_at"]) def downgrade() -> None: - op.drop_index(op.f("ix_auth_sessions_revoked_at"), table_name="auth_sessions") - op.drop_index(op.f("ix_auth_sessions_expires_at"), table_name="auth_sessions") - op.drop_index(op.f("ix_auth_sessions_token_hash"), table_name="auth_sessions") - op.drop_index(op.f("ix_auth_sessions_user_id"), table_name="auth_sessions") - op.drop_index(op.f("ix_auth_sessions_tenant_id"), table_name="auth_sessions") - op.drop_table("auth_sessions") + op.drop_index(op.f("ix_access_auth_sessions_revoked_at"), table_name="access_auth_sessions") + op.drop_index(op.f("ix_access_auth_sessions_expires_at"), table_name="access_auth_sessions") + op.drop_index(op.f("ix_access_auth_sessions_token_hash"), table_name="access_auth_sessions") + op.drop_index(op.f("ix_access_auth_sessions_user_id"), table_name="access_auth_sessions") + op.drop_index(op.f("ix_access_auth_sessions_tenant_id"), table_name="access_auth_sessions") + op.drop_table("access_auth_sessions") - op.drop_index(op.f("ix_group_role_assignments_role_id"), table_name="group_role_assignments") - op.drop_index(op.f("ix_group_role_assignments_group_id"), table_name="group_role_assignments") - op.drop_index(op.f("ix_group_role_assignments_tenant_id"), table_name="group_role_assignments") - op.drop_table("group_role_assignments") + op.drop_index(op.f("ix_access_group_role_assignments_role_id"), table_name="access_group_role_assignments") + op.drop_index(op.f("ix_access_group_role_assignments_group_id"), table_name="access_group_role_assignments") + op.drop_index(op.f("ix_access_group_role_assignments_tenant_id"), table_name="access_group_role_assignments") + op.drop_table("access_group_role_assignments") - op.drop_index(op.f("ix_user_role_assignments_role_id"), table_name="user_role_assignments") - op.drop_index(op.f("ix_user_role_assignments_user_id"), table_name="user_role_assignments") - op.drop_index(op.f("ix_user_role_assignments_tenant_id"), table_name="user_role_assignments") - op.drop_table("user_role_assignments") + op.drop_index(op.f("ix_access_user_role_assignments_role_id"), table_name="access_user_role_assignments") + op.drop_index(op.f("ix_access_user_role_assignments_user_id"), table_name="access_user_role_assignments") + op.drop_index(op.f("ix_access_user_role_assignments_tenant_id"), table_name="access_user_role_assignments") + op.drop_table("access_user_role_assignments") - op.drop_index(op.f("ix_user_group_memberships_group_id"), table_name="user_group_memberships") - op.drop_index(op.f("ix_user_group_memberships_user_id"), table_name="user_group_memberships") - op.drop_index(op.f("ix_user_group_memberships_tenant_id"), table_name="user_group_memberships") - op.drop_table("user_group_memberships") + op.drop_index(op.f("ix_access_user_group_memberships_group_id"), table_name="access_user_group_memberships") + op.drop_index(op.f("ix_access_user_group_memberships_user_id"), table_name="access_user_group_memberships") + op.drop_index(op.f("ix_access_user_group_memberships_tenant_id"), table_name="access_user_group_memberships") + op.drop_table("access_user_group_memberships") - with op.batch_alter_table("users") as batch_op: + with op.batch_alter_table("access_users") as batch_op: batch_op.drop_column("last_login_at") batch_op.drop_column("password_hash") batch_op.drop_column("auth_provider") diff --git a/alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py b/alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py new file mode 100644 index 0000000..093b258 --- /dev/null +++ b/alembic/versions/2e3f4a5b6c7d_namespace_platform_tables.py @@ -0,0 +1,78 @@ +"""namespace platform-owned tables + +Revision ID: 2e3f4a5b6c7d +Revises: 1b2c3d4e5f70 +Create Date: 2026-07-09 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "2e3f4a5b6c7d" +down_revision = "1b2c3d4e5f70" +branch_labels = None +depends_on = None + +_TABLE_RENAMES = ( + ("tenants", "tenancy_tenants"), + ("accounts", "access_accounts"), + ("users", "access_users"), + ("groups", "access_groups"), + ("roles", "access_roles"), + ("system_role_assignments", "access_system_role_assignments"), + ("user_group_memberships", "access_user_group_memberships"), + ("user_role_assignments", "access_user_role_assignments"), + ("group_role_assignments", "access_group_role_assignments"), + ("api_keys", "access_api_keys"), + ("auth_sessions", "access_auth_sessions"), + ("system_settings", "core_system_settings"), + ("governance_templates", "admin_governance_templates"), + ("governance_template_assignments", "admin_governance_template_assignments"), +) + + +def _row_count(bind: sa.Connection, table_name: str) -> int: + return int(bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one()) + + +def _rename_tables(renames: tuple[tuple[str, str], ...]) -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + tables = set(inspector.get_table_names()) + + old_tables_to_drop: set[str] = set() + new_tables_to_drop: set[str] = set() + for old_name, new_name in renames: + if old_name not in tables or new_name not in tables: + continue + if _row_count(bind, old_name) == 0: + old_tables_to_drop.add(old_name) + elif _row_count(bind, new_name) == 0: + new_tables_to_drop.add(new_name) + else: + raise RuntimeError(f"Cannot rename non-empty {old_name} over non-empty {new_name}") + + for old_name, _new_name in reversed(renames): + if old_name in old_tables_to_drop: + op.drop_table(old_name) + tables.remove(old_name) + for _old_name, new_name in reversed(renames): + if new_name in new_tables_to_drop: + op.drop_table(new_name) + tables.remove(new_name) + + for old_name, new_name in renames: + if old_name not in tables: + continue + op.rename_table(old_name, new_name) + tables.remove(old_name) + tables.add(new_name) + + +def upgrade() -> None: + _rename_tables(_TABLE_RENAMES) + + +def downgrade() -> None: + _rename_tables(tuple((new_name, old_name) for old_name, new_name in reversed(_TABLE_RENAMES))) diff --git a/alembic/versions/3d4e5f6a7b8c_file_storage_backend.py b/alembic/versions/3d4e5f6a7b8c_file_storage_backend.py index 862da74..01dee53 100644 --- a/alembic/versions/3d4e5f6a7b8c_file_storage_backend.py +++ b/alembic/versions/3d4e5f6a7b8c_file_storage_backend.py @@ -32,7 +32,7 @@ def upgrade() -> None: sa.Column("retained_until", sa.DateTime(timezone=True), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("tenant_id", "checksum_sha256", "size_bytes", name="uq_file_blobs_tenant_checksum_size"), ) @@ -55,10 +55,10 @@ def upgrade() -> None: sa.Column("metadata", sa.JSON(), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["owner_group_id"], ["groups.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), ) for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "current_version_id", "display_path", "filename", "created_by_user_id", "deleted_at"]: @@ -80,9 +80,9 @@ def upgrade() -> None: sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), sa.ForeignKeyConstraint(["blob_id"], ["file_blobs.id"], ondelete="RESTRICT"), - sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"), sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("file_asset_id", "version_number", name="uq_file_versions_asset_number"), ) @@ -101,9 +101,9 @@ def upgrade() -> None: sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"), sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("file_asset_id", "target_type", "target_id", "revoked_at", name="uq_file_shares_active_target"), ) @@ -136,7 +136,7 @@ def upgrade() -> None: sa.ForeignKeyConstraint(["file_asset_id"], ["file_assets.id"], ondelete="RESTRICT"), sa.ForeignKeyConstraint(["file_blob_id"], ["file_blobs.id"], ondelete="RESTRICT"), sa.ForeignKeyConstraint(["file_version_id"], ["file_versions.id"], ondelete="RESTRICT"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("campaign_job_id", "file_version_id", "filename_used", "use_stage", name="uq_campaign_attachment_uses_job_file_stage"), ) diff --git a/alembic/versions/3f4a5b6c7d8e_core_change_sequence.py b/alembic/versions/3f4a5b6c7d8e_core_change_sequence.py new file mode 100644 index 0000000..854a220 --- /dev/null +++ b/alembic/versions/3f4a5b6c7d8e_core_change_sequence.py @@ -0,0 +1,111 @@ +"""add core change sequence + +Revision ID: 3f4a5b6c7d8e +Revises: 2e3f4a5b6c7d +Create Date: 2026-07-09 00:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "3f4a5b6c7d8e" +down_revision = "2e3f4a5b6c7d" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "core_change_sequence" not in inspector.get_table_names(): + op.create_table( + "core_change_sequence", + sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=True), + sa.Column("module_id", sa.String(length=100), nullable=False), + sa.Column("collection", sa.String(length=150), nullable=False), + sa.Column("resource_type", sa.String(length=100), nullable=False), + sa.Column("resource_id", sa.String(length=255), nullable=False), + sa.Column("operation", sa.String(length=30), nullable=False), + sa.Column("actor_type", sa.String(length=30), nullable=True), + sa.Column("actor_id", sa.String(length=255), nullable=True), + sa.Column("payload", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_core_change_sequence")), + ) + + inspector = sa.inspect(op.get_bind()) + indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence")} + for column in ( + "tenant_id", + "module_id", + "collection", + "resource_type", + "resource_id", + "operation", + "actor_type", + "actor_id", + "created_at", + ): + name = op.f(f"ix_core_change_sequence_{column}") + if name not in indexes: + op.create_index(name, "core_change_sequence", [column], unique=False) + for name, columns in ( + ("ix_core_change_sequence_tenant_module_id", ["tenant_id", "module_id", "id"]), + ("ix_core_change_sequence_collection_id", ["collection", "id"]), + ("ix_core_change_sequence_resource", ["module_id", "resource_type", "resource_id"]), + ): + if name not in indexes: + op.create_index(name, "core_change_sequence", columns, unique=False) + + inspector = sa.inspect(op.get_bind()) + if "core_change_sequence_retention_floor" not in inspector.get_table_names(): + op.create_table( + "core_change_sequence_retention_floor", + sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False), + sa.Column("tenant_key", sa.String(length=36), nullable=False), + sa.Column("module_id", sa.String(length=100), nullable=False), + sa.Column("collection", sa.String(length=150), nullable=False), + sa.Column("min_valid_sequence", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_core_change_sequence_retention_floor")), + sa.UniqueConstraint("tenant_key", "module_id", "collection", name="uq_core_change_sequence_retention_scope"), + ) + indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence_retention_floor")} + if "ix_core_change_sequence_retention_scope" not in indexes: + op.create_index( + "ix_core_change_sequence_retention_scope", + "core_change_sequence_retention_floor", + ["tenant_key", "module_id", "collection"], + unique=False, + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if "core_change_sequence_retention_floor" in inspector.get_table_names(): + indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence_retention_floor")} + if "ix_core_change_sequence_retention_scope" in indexes: + op.drop_index("ix_core_change_sequence_retention_scope", table_name="core_change_sequence_retention_floor") + op.drop_table("core_change_sequence_retention_floor") + if "core_change_sequence" not in inspector.get_table_names(): + return + indexes = {item["name"] for item in inspector.get_indexes("core_change_sequence")} + for name in ( + "ix_core_change_sequence_resource", + "ix_core_change_sequence_collection_id", + "ix_core_change_sequence_tenant_module_id", + op.f("ix_core_change_sequence_created_at"), + op.f("ix_core_change_sequence_actor_id"), + op.f("ix_core_change_sequence_actor_type"), + op.f("ix_core_change_sequence_operation"), + op.f("ix_core_change_sequence_resource_id"), + op.f("ix_core_change_sequence_resource_type"), + op.f("ix_core_change_sequence_collection"), + op.f("ix_core_change_sequence_module_id"), + op.f("ix_core_change_sequence_tenant_id"), + ): + if name in indexes: + op.drop_index(name, table_name="core_change_sequence") + op.drop_table("core_change_sequence") diff --git a/alembic/versions/4e5f6a7b8c9d_file_folders.py b/alembic/versions/4e5f6a7b8c9d_file_folders.py index afed0f3..bd63263 100644 --- a/alembic/versions/4e5f6a7b8c9d_file_folders.py +++ b/alembic/versions/4e5f6a7b8c9d_file_folders.py @@ -31,10 +31,10 @@ def upgrade() -> None: sa.Column("metadata", sa.JSON(), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["owner_group_id"], ["groups.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["owner_user_id"], ["users.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["owner_group_id"], ["access_groups.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["owner_user_id"], ["access_users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), ) for col in ["tenant_id", "owner_type", "owner_user_id", "owner_group_id", "path", "created_by_user_id", "deleted_at"]: diff --git a/alembic/versions/5f6a7b8c9d0e_campaign_version_user_locks.py b/alembic/versions/5f6a7b8c9d0e_campaign_version_user_locks.py index 55ec532..678d747 100644 --- a/alembic/versions/5f6a7b8c9d0e_campaign_version_user_locks.py +++ b/alembic/versions/5f6a7b8c9d0e_campaign_version_user_locks.py @@ -24,7 +24,7 @@ def upgrade() -> None: batch_op.add_column(sa.Column("user_locked_by_user_id", sa.String(length=36), nullable=True)) batch_op.create_foreign_key( "fk_campaign_versions_user_locked_by_user_id_users", - "users", + "access_users", ["user_locked_by_user_id"], ["id"], ondelete="SET NULL", diff --git a/alembic/versions/8c9d0e1f2a3b_accounts_tenant_admin_rbac.py b/alembic/versions/8c9d0e1f2a3b_accounts_tenant_admin_rbac.py index 6c90506..0378f88 100644 --- a/alembic/versions/8c9d0e1f2a3b_accounts_tenant_admin_rbac.py +++ b/alembic/versions/8c9d0e1f2a3b_accounts_tenant_admin_rbac.py @@ -62,25 +62,25 @@ def upgrade() -> None: bind = op.get_bind() inspector = sa.inspect(bind) tables = set(inspector.get_table_names()) - user_columns = {column["name"] for column in inspector.get_columns("users")} + user_columns = {column["name"] for column in inspector.get_columns("access_users")} # Base.metadata.create_all() from a newer application can create brand-new # tables while leaving existing tables unaltered. Repair that known drift by # removing only empty, unreferenced administration tables before applying the # real migration. A non-empty table is never guessed at or discarded. - if "account_id" not in user_columns and "system_role_assignments" in tables: - count = bind.execute(sa.text("SELECT COUNT(*) FROM system_role_assignments")).scalar_one() + if "account_id" not in user_columns and "access_system_role_assignments" in tables: + count = bind.execute(sa.text("SELECT COUNT(*) FROM access_system_role_assignments")).scalar_one() if count: raise RuntimeError("Cannot reconcile non-empty create_all system_role_assignments table") - op.drop_table("system_role_assignments") - tables.remove("system_role_assignments") - if "account_id" not in user_columns and "accounts" in tables: - count = bind.execute(sa.text("SELECT COUNT(*) FROM accounts")).scalar_one() + op.drop_table("access_system_role_assignments") + tables.remove("access_system_role_assignments") + if "account_id" not in user_columns and "access_accounts" in tables: + count = bind.execute(sa.text("SELECT COUNT(*) FROM access_accounts")).scalar_one() if count: raise RuntimeError("Cannot reconcile non-empty create_all accounts table") - op.drop_table("accounts") + op.drop_table("access_accounts") op.create_table( - "accounts", + "access_accounts", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("email", sa.String(length=320), nullable=False), sa.Column("normalized_email", sa.String(length=320), nullable=False), @@ -95,29 +95,29 @@ def upgrade() -> None: sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("normalized_email", name="uq_accounts_normalized_email"), ) - op.create_index(op.f("ix_accounts_normalized_email"), "accounts", ["normalized_email"]) + op.create_index(op.f("ix_access_accounts_normalized_email"), "access_accounts", ["normalized_email"]) - with op.batch_alter_table("tenants") as batch_op: + with op.batch_alter_table("tenancy_tenants") as batch_op: batch_op.add_column(sa.Column("description", sa.Text(), nullable=True)) batch_op.add_column(sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en")) batch_op.add_column(sa.Column("settings", sa.JSON(), nullable=False, server_default="{}")) - with op.batch_alter_table("groups") as batch_op: + with op.batch_alter_table("access_groups") as batch_op: batch_op.add_column(sa.Column("description", sa.Text(), nullable=True)) batch_op.add_column(sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true())) - with op.batch_alter_table("roles") as batch_op: + with op.batch_alter_table("access_roles") as batch_op: batch_op.add_column(sa.Column("description", sa.Text(), nullable=True)) batch_op.add_column(sa.Column("is_builtin", sa.Boolean(), nullable=False, server_default=sa.false())) batch_op.add_column(sa.Column("is_assignable", sa.Boolean(), nullable=False, server_default=sa.true())) - with op.batch_alter_table("users") as batch_op: + with op.batch_alter_table("access_users") as batch_op: batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True)) - with op.batch_alter_table("auth_sessions") as batch_op: + with op.batch_alter_table("access_auth_sessions") as batch_op: batch_op.add_column(sa.Column("account_id", sa.String(length=36), nullable=True)) users = bind.execute(sa.text( - "SELECT id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at FROM users ORDER BY created_at, id" + "SELECT id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at FROM access_users ORDER BY created_at, id" )).mappings().all() accounts_by_email: dict[str, str] = {} @@ -155,7 +155,7 @@ def upgrade() -> None: first_system_owner_account_id = account_id accounts_table = sa.table( - "accounts", + "access_accounts", sa.column("id", sa.String), sa.column("email", sa.String), sa.column("normalized_email", sa.String), sa.column("display_name", sa.String), sa.column("is_active", sa.Boolean), sa.column("auth_provider", sa.String), sa.column("password_hash", sa.String), sa.column("password_reset_required", sa.Boolean), @@ -166,54 +166,54 @@ def upgrade() -> None: bind.execute(accounts_table.insert(), list(account_rows.values())) for row in users: bind.execute( - sa.text("UPDATE users SET account_id = :account_id WHERE id = :user_id"), + sa.text("UPDATE access_users SET account_id = :account_id WHERE id = :user_id"), {"account_id": accounts_by_email[_normalize_email(row["email"])], "user_id": row["id"]}, ) bind.execute(sa.text( - "UPDATE auth_sessions SET account_id = (SELECT users.account_id FROM users WHERE users.id = auth_sessions.user_id)" + "UPDATE access_auth_sessions SET account_id = (SELECT access_users.account_id FROM access_users WHERE access_users.id = access_auth_sessions.user_id)" )) - with op.batch_alter_table("users") as batch_op: + with op.batch_alter_table("access_users") as batch_op: batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False) - batch_op.create_foreign_key("fk_users_account_id_accounts", "accounts", ["account_id"], ["id"], ondelete="CASCADE") + batch_op.create_foreign_key("fk_users_account_id_accounts", "access_accounts", ["account_id"], ["id"], ondelete="CASCADE") batch_op.create_unique_constraint("uq_users_tenant_account", ["tenant_id", "account_id"]) - op.create_index(op.f("ix_users_account_id"), "users", ["account_id"]) + op.create_index(op.f("ix_access_users_account_id"), "access_users", ["account_id"]) - with op.batch_alter_table("auth_sessions") as batch_op: + with op.batch_alter_table("access_auth_sessions") as batch_op: batch_op.alter_column("account_id", existing_type=sa.String(length=36), nullable=False) - batch_op.create_foreign_key("fk_auth_sessions_account_id_accounts", "accounts", ["account_id"], ["id"], ondelete="CASCADE") - op.create_index(op.f("ix_auth_sessions_account_id"), "auth_sessions", ["account_id"]) + batch_op.create_foreign_key("fk_auth_sessions_account_id_accounts", "access_accounts", ["account_id"], ["id"], ondelete="CASCADE") + op.create_index(op.f("ix_access_auth_sessions_account_id"), "access_auth_sessions", ["account_id"]) op.create_table( - "system_role_assignments", + "access_system_role_assignments", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("account_id", sa.String(length=36), nullable=False), sa.Column("role_id", sa.String(length=36), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["account_id"], ["accounts.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["role_id"], ["roles.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["account_id"], ["access_accounts.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["role_id"], ["access_roles.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("account_id", "role_id", name="uq_system_role_assignments"), ) - op.create_index(op.f("ix_system_role_assignments_account_id"), "system_role_assignments", ["account_id"]) - op.create_index(op.f("ix_system_role_assignments_role_id"), "system_role_assignments", ["role_id"]) + op.create_index(op.f("ix_access_system_role_assignments_account_id"), "access_system_role_assignments", ["account_id"]) + op.create_index(op.f("ix_access_system_role_assignments_role_id"), "access_system_role_assignments", ["role_id"]) roles_table = sa.table( - "roles", + "access_roles", sa.column("id", sa.String), sa.column("tenant_id", sa.String), sa.column("slug", sa.String), sa.column("name", sa.String), sa.column("description", sa.Text), sa.column("permissions", sa.JSON), sa.column("is_builtin", sa.Boolean), sa.column("is_assignable", sa.Boolean), sa.column("created_at", sa.DateTime(timezone=True)), sa.column("updated_at", sa.DateTime(timezone=True)), ) now = _now() - tenant_ids = [row[0] for row in bind.execute(sa.text("SELECT id FROM tenants")).all()] + tenant_ids = [row[0] for row in bind.execute(sa.text("SELECT id FROM tenancy_tenants")).all()] definitions = _role_definitions() tenant_role_ids: dict[tuple[str, str], str] = {} for tenant_id in tenant_ids: for slug, definition in definitions.items(): existing = bind.execute( - sa.text("SELECT id FROM roles WHERE tenant_id = :tenant_id AND slug = :slug"), + sa.text("SELECT id FROM access_roles WHERE tenant_id = :tenant_id AND slug = :slug"), {"tenant_id": tenant_id, "slug": slug}, ).scalar_one_or_none() if existing: @@ -253,7 +253,7 @@ def upgrade() -> None: op.create_index( "uq_roles_system_slug", - "roles", + "access_roles", ["slug"], unique=True, sqlite_where=sa.text("tenant_id IS NULL"), @@ -267,11 +267,11 @@ def upgrade() -> None: continue owner_role_id = tenant_role_ids[(row["tenant_id"], "owner")] exists = bind.execute(sa.text( - "SELECT 1 FROM user_role_assignments WHERE tenant_id = :tenant_id AND user_id = :user_id AND role_id = :role_id" + "SELECT 1 FROM access_user_role_assignments WHERE tenant_id = :tenant_id AND user_id = :user_id AND role_id = :role_id" ), {"tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id}).first() if not exists: bind.execute(sa.text( - "INSERT INTO user_role_assignments (id, tenant_id, user_id, role_id, created_at, updated_at) VALUES (:id, :tenant_id, :user_id, :role_id, :created_at, :updated_at)" + "INSERT INTO access_user_role_assignments (id, tenant_id, user_id, role_id, created_at, updated_at) VALUES (:id, :tenant_id, :user_id, :role_id, :created_at, :updated_at)" ), {"id": str(uuid.uuid4()), "tenant_id": row["tenant_id"], "user_id": row["id"], "role_id": owner_role_id, "created_at": now, "updated_at": now}) # Bootstrap rule for existing installations: the earliest active legacy @@ -284,40 +284,40 @@ def upgrade() -> None: ), None) if first_system_owner_account_id: bind.execute(sa.text( - "INSERT INTO system_role_assignments (id, account_id, role_id, created_at, updated_at) VALUES (:id, :account_id, :role_id, :created_at, :updated_at)" + "INSERT INTO access_system_role_assignments (id, account_id, role_id, created_at, updated_at) VALUES (:id, :account_id, :role_id, :created_at, :updated_at)" ), {"id": str(uuid.uuid4()), "account_id": first_system_owner_account_id, "role_id": system_owner_role_id, "created_at": now, "updated_at": now}) def downgrade() -> None: - op.drop_index("uq_roles_system_slug", table_name="roles") - op.drop_index(op.f("ix_system_role_assignments_role_id"), table_name="system_role_assignments") - op.drop_index(op.f("ix_system_role_assignments_account_id"), table_name="system_role_assignments") - op.drop_table("system_role_assignments") + op.drop_index("uq_roles_system_slug", table_name="access_roles") + op.drop_index(op.f("ix_access_system_role_assignments_role_id"), table_name="access_system_role_assignments") + op.drop_index(op.f("ix_access_system_role_assignments_account_id"), table_name="access_system_role_assignments") + op.drop_table("access_system_role_assignments") - op.drop_index(op.f("ix_auth_sessions_account_id"), table_name="auth_sessions") - with op.batch_alter_table("auth_sessions") as batch_op: + op.drop_index(op.f("ix_access_auth_sessions_account_id"), table_name="access_auth_sessions") + with op.batch_alter_table("access_auth_sessions") as batch_op: batch_op.drop_constraint("fk_auth_sessions_account_id_accounts", type_="foreignkey") batch_op.drop_column("account_id") - op.drop_index(op.f("ix_users_account_id"), table_name="users") - with op.batch_alter_table("users") as batch_op: + op.drop_index(op.f("ix_access_users_account_id"), table_name="access_users") + with op.batch_alter_table("access_users") as batch_op: batch_op.drop_constraint("uq_users_tenant_account", type_="unique") batch_op.drop_constraint("fk_users_account_id_accounts", type_="foreignkey") batch_op.drop_column("account_id") - with op.batch_alter_table("roles") as batch_op: + with op.batch_alter_table("access_roles") as batch_op: batch_op.drop_column("is_assignable") batch_op.drop_column("is_builtin") batch_op.drop_column("description") - with op.batch_alter_table("groups") as batch_op: + with op.batch_alter_table("access_groups") as batch_op: batch_op.drop_column("is_active") batch_op.drop_column("description") - with op.batch_alter_table("tenants") as batch_op: + with op.batch_alter_table("tenancy_tenants") as batch_op: batch_op.drop_column("settings") batch_op.drop_column("default_locale") batch_op.drop_column("description") - op.drop_index(op.f("ix_accounts_normalized_email"), table_name="accounts") - op.drop_table("accounts") + op.drop_index(op.f("ix_access_accounts_normalized_email"), table_name="access_accounts") + op.drop_table("access_accounts") diff --git a/alembic/versions/9d0e1f2a3b4c_system_governance_templates.py b/alembic/versions/9d0e1f2a3b4c_system_governance_templates.py index ce06118..963189a 100644 --- a/alembic/versions/9d0e1f2a3b4c_system_governance_templates.py +++ b/alembic/versions/9d0e1f2a3b4c_system_governance_templates.py @@ -28,7 +28,7 @@ def upgrade() -> None: tables = set(inspector.get_table_names()) # Reconcile only the empty create_all shape for the newly introduced tables. - for table_name in ("governance_template_assignments", "governance_templates", "system_settings"): + for table_name in ("admin_governance_template_assignments", "admin_governance_templates", "core_system_settings"): if table_name in tables: count = bind.execute(sa.text(f"SELECT COUNT(*) FROM {table_name}")).scalar_one() if count: @@ -36,7 +36,7 @@ def upgrade() -> None: op.drop_table(table_name) op.create_table( - "system_settings", + "core_system_settings", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("default_locale", sa.String(length=20), nullable=False, server_default="en"), sa.Column("allow_tenant_custom_groups", sa.Boolean(), nullable=False, server_default=sa.true()), @@ -50,16 +50,23 @@ def upgrade() -> None: now = _now() bind.execute(sa.text( """ - INSERT INTO system_settings + INSERT INTO core_system_settings (id, default_locale, allow_tenant_custom_groups, allow_tenant_custom_roles, allow_tenant_api_keys, settings, created_at, updated_at) VALUES - ('global', 'en', 1, 1, 1, '{}', :created_at, :updated_at) + ('global', 'en', :allow_tenant_custom_groups, :allow_tenant_custom_roles, + :allow_tenant_api_keys, '{}', :created_at, :updated_at) """ - ), {"created_at": now, "updated_at": now}) + ), { + "allow_tenant_custom_groups": True, + "allow_tenant_custom_roles": True, + "allow_tenant_api_keys": True, + "created_at": now, + "updated_at": now, + }) op.create_table( - "governance_templates", + "admin_governance_templates", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("kind", sa.String(length=20), nullable=False), sa.Column("slug", sa.String(length=100), nullable=False), @@ -72,51 +79,51 @@ def upgrade() -> None: sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("kind", "slug", name="uq_governance_templates_kind_slug"), ) - op.create_index(op.f("ix_governance_templates_kind"), "governance_templates", ["kind"]) + op.create_index(op.f("ix_admin_governance_templates_kind"), "admin_governance_templates", ["kind"]) op.create_table( - "governance_template_assignments", + "admin_governance_template_assignments", sa.Column("id", sa.String(length=36), nullable=False), sa.Column("template_id", sa.String(length=36), nullable=False), sa.Column("tenant_id", sa.String(length=36), nullable=False), sa.Column("mode", sa.String(length=20), nullable=False, server_default="available"), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["template_id"], ["governance_templates.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["template_id"], ["admin_governance_templates.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("template_id", "tenant_id", name="uq_governance_template_tenant"), ) - op.create_index(op.f("ix_governance_template_assignments_template_id"), "governance_template_assignments", ["template_id"]) - op.create_index(op.f("ix_governance_template_assignments_tenant_id"), "governance_template_assignments", ["tenant_id"]) + op.create_index(op.f("ix_admin_governance_template_assignments_template_id"), "admin_governance_template_assignments", ["template_id"]) + op.create_index(op.f("ix_admin_governance_template_assignments_tenant_id"), "admin_governance_template_assignments", ["tenant_id"]) - with op.batch_alter_table("tenants") as batch_op: + with op.batch_alter_table("tenancy_tenants") as batch_op: batch_op.add_column(sa.Column("allow_custom_groups", sa.Boolean(), nullable=True)) batch_op.add_column(sa.Column("allow_custom_roles", sa.Boolean(), nullable=True)) batch_op.add_column(sa.Column("allow_api_keys", sa.Boolean(), nullable=True)) - with op.batch_alter_table("groups") as batch_op: + with op.batch_alter_table("access_groups") as batch_op: batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True)) batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false())) batch_op.create_foreign_key( "fk_groups_system_template_id_governance_templates", - "governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL", + "admin_governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL", ) - op.create_index(op.f("ix_groups_system_template_id"), "groups", ["system_template_id"]) + op.create_index(op.f("ix_access_groups_system_template_id"), "access_groups", ["system_template_id"]) - with op.batch_alter_table("roles") as batch_op: + with op.batch_alter_table("access_roles") as batch_op: batch_op.add_column(sa.Column("system_template_id", sa.String(length=36), nullable=True)) batch_op.add_column(sa.Column("system_required", sa.Boolean(), nullable=False, server_default=sa.false())) batch_op.create_foreign_key( "fk_roles_system_template_id_governance_templates", - "governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL", + "admin_governance_templates", ["system_template_id"], ["id"], ondelete="SET NULL", ) - op.create_index(op.f("ix_roles_system_template_id"), "roles", ["system_template_id"]) + op.create_index(op.f("ix_access_roles_system_template_id"), "access_roles", ["system_template_id"]) # Existing system owners use system:* and need no data change. Extend the # read-only built-in auditor role to the newly introduced read scopes. auditor = bind.execute(sa.text( - "SELECT id, permissions FROM roles WHERE tenant_id IS NULL AND slug = 'system_auditor'" + "SELECT id, permissions FROM access_roles WHERE tenant_id IS NULL AND slug = 'system_auditor'" )).mappings().first() if auditor: raw_permissions = auditor["permissions"] or [] @@ -125,7 +132,7 @@ def upgrade() -> None: if scope not in permissions: permissions.append(scope) roles_table = sa.table( - "roles", + "access_roles", sa.column("id", sa.String), sa.column("permissions", sa.JSON), sa.column("updated_at", sa.DateTime(timezone=True)), @@ -138,26 +145,26 @@ def upgrade() -> None: def downgrade() -> None: - op.drop_index(op.f("ix_roles_system_template_id"), table_name="roles") - with op.batch_alter_table("roles") as batch_op: + op.drop_index(op.f("ix_access_roles_system_template_id"), table_name="access_roles") + with op.batch_alter_table("access_roles") as batch_op: batch_op.drop_constraint("fk_roles_system_template_id_governance_templates", type_="foreignkey") batch_op.drop_column("system_required") batch_op.drop_column("system_template_id") - op.drop_index(op.f("ix_groups_system_template_id"), table_name="groups") - with op.batch_alter_table("groups") as batch_op: + op.drop_index(op.f("ix_access_groups_system_template_id"), table_name="access_groups") + with op.batch_alter_table("access_groups") as batch_op: batch_op.drop_constraint("fk_groups_system_template_id_governance_templates", type_="foreignkey") batch_op.drop_column("system_required") batch_op.drop_column("system_template_id") - with op.batch_alter_table("tenants") as batch_op: + with op.batch_alter_table("tenancy_tenants") as batch_op: batch_op.drop_column("allow_api_keys") batch_op.drop_column("allow_custom_roles") batch_op.drop_column("allow_custom_groups") - op.drop_index(op.f("ix_governance_template_assignments_tenant_id"), table_name="governance_template_assignments") - op.drop_index(op.f("ix_governance_template_assignments_template_id"), table_name="governance_template_assignments") - op.drop_table("governance_template_assignments") - op.drop_index(op.f("ix_governance_templates_kind"), table_name="governance_templates") - op.drop_table("governance_templates") - op.drop_table("system_settings") + op.drop_index(op.f("ix_admin_governance_template_assignments_tenant_id"), table_name="admin_governance_template_assignments") + op.drop_index(op.f("ix_admin_governance_template_assignments_template_id"), table_name="admin_governance_template_assignments") + op.drop_table("admin_governance_template_assignments") + op.drop_index(op.f("ix_admin_governance_templates_kind"), table_name="admin_governance_templates") + op.drop_table("admin_governance_templates") + op.drop_table("core_system_settings") diff --git a/alembic/versions/a0b1c2d3e4f5_permission_catalogue_refinement.py b/alembic/versions/a0b1c2d3e4f5_permission_catalogue_refinement.py index b199b6c..c0c8262 100644 --- a/alembic/versions/a0b1c2d3e4f5_permission_catalogue_refinement.py +++ b/alembic/versions/a0b1c2d3e4f5_permission_catalogue_refinement.py @@ -148,9 +148,9 @@ def upgrade() -> None: sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), sa.ForeignKeyConstraint(["campaign_id"], ["campaigns.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("campaign_id", "target_type", "target_id", name="uq_campaign_share_target"), ) @@ -160,32 +160,32 @@ def upgrade() -> None: op.create_index("ix_campaign_shares_target_id", "campaign_shares", ["target_id"]) op.create_index("ix_campaign_shares_created_by_user_id", "campaign_shares", ["created_by_user_id"]) op.create_index("ix_campaign_shares_revoked_at", "campaign_shares", ["revoked_at"]) - if "roles" not in tables: + if "access_roles" not in tables: return - rows = bind.execute(sa.text("SELECT id, permissions FROM roles")).mappings().all() + rows = bind.execute(sa.text("SELECT id, permissions FROM access_roles")).mappings().all() for row in rows: bind.execute( - sa.text("UPDATE roles SET permissions = :permissions WHERE id = :id"), + sa.text("UPDATE access_roles SET permissions = :permissions WHERE id = :id"), {"id": row["id"], "permissions": _json(_expand_legacy(_decode(row["permissions"])))}, ) for slug, permissions in TENANT_ROLE_PERMISSIONS.items(): bind.execute( - sa.text("UPDATE roles SET permissions = :permissions WHERE tenant_id IS NOT NULL AND slug = :slug AND is_builtin = TRUE"), + sa.text("UPDATE access_roles SET permissions = :permissions WHERE tenant_id IS NOT NULL AND slug = :slug AND is_builtin = TRUE"), {"slug": slug, "permissions": _json(permissions)}, ) now = datetime.now(timezone.utc) for slug, permissions in SYSTEM_ROLE_PERMISSIONS.items(): existing = bind.execute( - sa.text("SELECT id FROM roles WHERE tenant_id IS NULL AND slug = :slug"), {"slug": slug} + sa.text("SELECT id FROM access_roles WHERE tenant_id IS NULL AND slug = :slug"), {"slug": slug} ).first() is_protected = slug == "system_owner" if existing: bind.execute( sa.text( - "UPDATE roles SET permissions = :permissions, is_builtin = :is_builtin, is_assignable = TRUE " + "UPDATE access_roles SET permissions = :permissions, is_builtin = :is_builtin, is_assignable = TRUE " "WHERE tenant_id IS NULL AND slug = :slug" ), {"slug": slug, "permissions": _json(permissions), "is_builtin": is_protected}, @@ -199,7 +199,7 @@ def upgrade() -> None: name, description = names[slug] bind.execute( sa.text( - "INSERT INTO roles (id, tenant_id, slug, name, description, permissions, is_builtin, is_assignable, " + "INSERT INTO access_roles (id, tenant_id, slug, name, description, permissions, is_builtin, is_assignable, " "system_template_id, system_required, created_at, updated_at) " "VALUES (:id, NULL, :slug, :name, :description, :permissions, :is_builtin, TRUE, NULL, FALSE, :created_at, :updated_at)" ), diff --git a/alembic/versions/b57c5b216bce_initial_persistence_models.py b/alembic/versions/b57c5b216bce_initial_persistence_models.py index 3f4f933..91163a3 100644 --- a/alembic/versions/b57c5b216bce_initial_persistence_models.py +++ b/alembic/versions/b57c5b216bce_initial_persistence_models.py @@ -18,7 +18,7 @@ depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('tenants', + op.create_table('tenancy_tenants', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('slug', sa.String(length=100), nullable=False), sa.Column('name', sa.String(length=255), nullable=False), @@ -27,7 +27,7 @@ def upgrade() -> None: sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), sa.PrimaryKeyConstraint('id', name=op.f('pk_tenants')) ) - op.create_index(op.f('ix_tenants_slug'), 'tenants', ['slug'], unique=True) + op.create_index(op.f('ix_tenancy_tenants_slug'), 'tenancy_tenants', ['slug'], unique=True) op.create_table('attachment_blobs', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=False), @@ -38,25 +38,25 @@ def upgrade() -> None: sa.Column('storage_key', sa.String(length=1000), nullable=False), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_attachment_blobs_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_attachment_blobs_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_blobs')), sa.UniqueConstraint('tenant_id', 'sha256', name='uq_attachment_blobs_tenant_sha256') ) op.create_index(op.f('ix_attachment_blobs_sha256'), 'attachment_blobs', ['sha256'], unique=False) op.create_index(op.f('ix_attachment_blobs_tenant_id'), 'attachment_blobs', ['tenant_id'], unique=False) - op.create_table('groups', + op.create_table('access_groups', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=False), sa.Column('slug', sa.String(length=100), nullable=False), sa.Column('name', sa.String(length=255), nullable=False), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_groups_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_groups_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_groups')), sa.UniqueConstraint('tenant_id', 'slug', name='uq_groups_tenant_slug') ) - op.create_index(op.f('ix_groups_tenant_id'), 'groups', ['tenant_id'], unique=False) - op.create_table('roles', + op.create_index(op.f('ix_access_groups_tenant_id'), 'access_groups', ['tenant_id'], unique=False) + op.create_table('access_roles', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('slug', sa.String(length=100), nullable=False), @@ -64,12 +64,12 @@ def upgrade() -> None: sa.Column('permissions', sa.JSON(), nullable=False), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_roles_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_roles_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_roles')), sa.UniqueConstraint('tenant_id', 'slug', name='uq_roles_tenant_slug') ) - op.create_index(op.f('ix_roles_tenant_id'), 'roles', ['tenant_id'], unique=False) - op.create_table('users', + op.create_index(op.f('ix_access_roles_tenant_id'), 'access_roles', ['tenant_id'], unique=False) + op.create_table('access_users', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=False), sa.Column('email', sa.String(length=320), nullable=False), @@ -78,13 +78,13 @@ def upgrade() -> None: sa.Column('is_tenant_admin', sa.Boolean(), nullable=False), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_users_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_users_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_users')), sa.UniqueConstraint('tenant_id', 'email', name='uq_users_tenant_email') ) - op.create_index(op.f('ix_users_email'), 'users', ['email'], unique=False) - op.create_index(op.f('ix_users_tenant_id'), 'users', ['tenant_id'], unique=False) - op.create_table('api_keys', + op.create_index(op.f('ix_access_users_email'), 'access_users', ['email'], unique=False) + op.create_index(op.f('ix_access_users_tenant_id'), 'access_users', ['tenant_id'], unique=False) + op.create_table('access_api_keys', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=False), sa.Column('user_id', sa.String(length=36), nullable=False), @@ -97,13 +97,13 @@ def upgrade() -> None: sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_api_keys_tenant_id_tenants'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_api_keys_user_id_users'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_api_keys_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_api_keys_user_id_users'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_api_keys')) ) - op.create_index(op.f('ix_api_keys_prefix'), 'api_keys', ['prefix'], unique=False) - op.create_index(op.f('ix_api_keys_tenant_id'), 'api_keys', ['tenant_id'], unique=False) - op.create_index(op.f('ix_api_keys_user_id'), 'api_keys', ['user_id'], unique=False) + op.create_index(op.f('ix_access_api_keys_prefix'), 'access_api_keys', ['prefix'], unique=False) + op.create_index(op.f('ix_access_api_keys_tenant_id'), 'access_api_keys', ['tenant_id'], unique=False) + op.create_index(op.f('ix_access_api_keys_user_id'), 'access_api_keys', ['user_id'], unique=False) op.create_table('campaigns', sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=False), @@ -115,8 +115,8 @@ def upgrade() -> None: sa.Column('current_version_id', sa.String(length=36), nullable=True), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['created_by_user_id'], ['users.id'], name=op.f('fk_campaigns_created_by_user_id_users'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaigns_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['created_by_user_id'], ['access_users.id'], name=op.f('fk_campaigns_created_by_user_id_users'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaigns_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_campaigns')), sa.UniqueConstraint('tenant_id', 'external_id', name='uq_campaigns_tenant_external_id') ) @@ -138,8 +138,8 @@ def upgrade() -> None: sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), sa.ForeignKeyConstraint(['blob_id'], ['attachment_blobs.id'], name=op.f('fk_attachment_instances_blob_id_attachment_blobs'), ondelete='CASCADE'), sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_attachment_instances_campaign_id_campaigns'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['owner_user_id'], ['users.id'], name=op.f('fk_attachment_instances_owner_user_id_users'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_attachment_instances_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['owner_user_id'], ['access_users.id'], name=op.f('fk_attachment_instances_owner_user_id_users'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_attachment_instances_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_attachment_instances')) ) op.create_index(op.f('ix_attachment_instances_blob_id'), 'attachment_instances', ['blob_id'], unique=False) @@ -157,9 +157,9 @@ def upgrade() -> None: sa.Column('details', sa.JSON(), nullable=True), sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(['api_key_id'], ['api_keys.id'], name=op.f('fk_audit_log_api_key_id_api_keys'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_audit_log_tenant_id_tenants'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('fk_audit_log_user_id_users'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['api_key_id'], ['access_api_keys.id'], name=op.f('fk_audit_log_api_key_id_api_keys'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_audit_log_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['access_users.id'], name=op.f('fk_audit_log_user_id_users'), ondelete='SET NULL'), sa.PrimaryKeyConstraint('id', name=op.f('pk_audit_log')) ) op.create_index(op.f('ix_audit_log_action'), 'audit_log', ['action'], unique=False) @@ -213,7 +213,7 @@ def upgrade() -> None: sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_jobs_campaign_id_campaigns'), ondelete='CASCADE'), sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_jobs_campaign_version_id_campaign_versions'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaign_jobs_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaign_jobs_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_jobs')), sa.UniqueConstraint('campaign_version_id', 'entry_index', name='uq_campaign_jobs_version_entry') ) @@ -243,7 +243,7 @@ def upgrade() -> None: sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.id'], name=op.f('fk_campaign_issues_campaign_id_campaigns'), ondelete='CASCADE'), sa.ForeignKeyConstraint(['campaign_version_id'], ['campaign_versions.id'], name=op.f('fk_campaign_issues_campaign_version_id_campaign_versions'), ondelete='CASCADE'), sa.ForeignKeyConstraint(['job_id'], ['campaign_jobs.id'], name=op.f('fk_campaign_issues_job_id_campaign_jobs'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], name=op.f('fk_campaign_issues_tenant_id_tenants'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tenant_id'], ['tenancy_tenants.id'], name=op.f('fk_campaign_issues_tenant_id_tenants'), ondelete='CASCADE'), sa.PrimaryKeyConstraint('id', name=op.f('pk_campaign_issues')) ) op.create_index(op.f('ix_campaign_issues_campaign_id'), 'campaign_issues', ['campaign_id'], unique=False) @@ -327,20 +327,20 @@ def downgrade() -> None: op.drop_index(op.f('ix_campaigns_external_id'), table_name='campaigns') op.drop_index(op.f('ix_campaigns_created_by_user_id'), table_name='campaigns') op.drop_table('campaigns') - op.drop_index(op.f('ix_api_keys_user_id'), table_name='api_keys') - op.drop_index(op.f('ix_api_keys_tenant_id'), table_name='api_keys') - op.drop_index(op.f('ix_api_keys_prefix'), table_name='api_keys') - op.drop_table('api_keys') - op.drop_index(op.f('ix_users_tenant_id'), table_name='users') - op.drop_index(op.f('ix_users_email'), table_name='users') - op.drop_table('users') - op.drop_index(op.f('ix_roles_tenant_id'), table_name='roles') - op.drop_table('roles') - op.drop_index(op.f('ix_groups_tenant_id'), table_name='groups') - op.drop_table('groups') + op.drop_index(op.f('ix_access_api_keys_user_id'), table_name='access_api_keys') + op.drop_index(op.f('ix_access_api_keys_tenant_id'), table_name='access_api_keys') + op.drop_index(op.f('ix_access_api_keys_prefix'), table_name='access_api_keys') + op.drop_table('access_api_keys') + op.drop_index(op.f('ix_access_users_tenant_id'), table_name='access_users') + op.drop_index(op.f('ix_access_users_email'), table_name='access_users') + op.drop_table('access_users') + op.drop_index(op.f('ix_access_roles_tenant_id'), table_name='access_roles') + op.drop_table('access_roles') + op.drop_index(op.f('ix_access_groups_tenant_id'), table_name='access_groups') + op.drop_table('access_groups') op.drop_index(op.f('ix_attachment_blobs_tenant_id'), table_name='attachment_blobs') op.drop_index(op.f('ix_attachment_blobs_sha256'), table_name='attachment_blobs') op.drop_table('attachment_blobs') - op.drop_index(op.f('ix_tenants_slug'), table_name='tenants') - op.drop_table('tenants') + op.drop_index(op.f('ix_tenancy_tenants_slug'), table_name='tenancy_tenants') + op.drop_table('tenancy_tenants') # ### end Alembic commands ### diff --git a/alembic/versions/d3e4f5a6b7c8_mail_profiles_and_cookie_csrf.py b/alembic/versions/d3e4f5a6b7c8_mail_profiles_and_cookie_csrf.py index 5b8985a..6f7957b 100644 --- a/alembic/versions/d3e4f5a6b7c8_mail_profiles_and_cookie_csrf.py +++ b/alembic/versions/d3e4f5a6b7c8_mail_profiles_and_cookie_csrf.py @@ -19,10 +19,10 @@ def upgrade() -> None: bind = op.get_bind() inspector = sa.inspect(bind) tables = set(inspector.get_table_names()) - if "auth_sessions" in tables: - columns = {column["name"] for column in inspector.get_columns("auth_sessions")} + if "access_auth_sessions" in tables: + columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")} if "csrf_token_hash" not in columns: - op.add_column("auth_sessions", sa.Column("csrf_token_hash", sa.String(length=128), nullable=True)) + op.add_column("access_auth_sessions", sa.Column("csrf_token_hash", sa.String(length=128), nullable=True)) if "mail_server_profiles" not in tables: op.create_table( "mail_server_profiles", @@ -40,9 +40,9 @@ def upgrade() -> None: sa.Column("updated_by_user_id", sa.String(length=36), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), - sa.ForeignKeyConstraint(["created_by_user_id"], ["users.id"], ondelete="SET NULL"), - sa.ForeignKeyConstraint(["tenant_id"], ["tenants.id"], ondelete="CASCADE"), - sa.ForeignKeyConstraint(["updated_by_user_id"], ["users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["created_by_user_id"], ["access_users.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["tenant_id"], ["tenancy_tenants.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["updated_by_user_id"], ["access_users.id"], ondelete="SET NULL"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("tenant_id", "slug", name="uq_mail_server_profiles_tenant_slug"), ) @@ -67,7 +67,7 @@ def downgrade() -> None: except Exception: pass op.drop_table("mail_server_profiles") - if "auth_sessions" in inspector.get_table_names(): - columns = {column["name"] for column in inspector.get_columns("auth_sessions")} + if "access_auth_sessions" in inspector.get_table_names(): + columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")} if "csrf_token_hash" in columns: - op.drop_column("auth_sessions", "csrf_token_hash") + op.drop_column("access_auth_sessions", "csrf_token_hash") diff --git a/alembic/versions/e4f5a6b7c8d9_mail_profile_policy_hierarchy.py b/alembic/versions/e4f5a6b7c8d9_mail_profile_policy_hierarchy.py index d51bf62..01cf39c 100644 --- a/alembic/versions/e4f5a6b7c8d9_mail_profile_policy_hierarchy.py +++ b/alembic/versions/e4f5a6b7c8d9_mail_profile_policy_hierarchy.py @@ -31,7 +31,7 @@ def upgrade() -> None: inspector = sa.inspect(bind) tables = set(inspector.get_table_names()) - for table_name in ("users", "groups", "campaigns"): + for table_name in ("access_users", "access_groups", "campaigns"): if table_name not in tables: continue columns = _columns(inspector, table_name) @@ -83,6 +83,6 @@ def downgrade() -> None: batch.drop_column("scope_type") batch.alter_column("tenant_id", existing_type=sa.String(length=36), nullable=False) - for table_name in ("campaigns", "groups", "users"): + for table_name in ("campaigns", "access_groups", "access_users"): if table_name in tables and "mail_profile_policy" in _columns(inspector, table_name): op.drop_column(table_name, "mail_profile_policy") diff --git a/alembic/versions/f5a6b7c8d9e0_hierarchical_settings.py b/alembic/versions/f5a6b7c8d9e0_hierarchical_settings.py index 5cc99f0..c88a04c 100644 --- a/alembic/versions/f5a6b7c8d9e0_hierarchical_settings.py +++ b/alembic/versions/f5a6b7c8d9e0_hierarchical_settings.py @@ -25,7 +25,7 @@ def upgrade() -> None: bind = op.get_bind() inspector = sa.inspect(bind) tables = set(inspector.get_table_names()) - for table_name in ("users", "groups", "campaigns"): + for table_name in ("access_users", "access_groups", "campaigns"): if table_name not in tables: continue if "settings" not in _columns(inspector, table_name): @@ -36,6 +36,6 @@ def downgrade() -> None: bind = op.get_bind() inspector = sa.inspect(bind) tables = set(inspector.get_table_names()) - for table_name in ("campaigns", "groups", "users"): + for table_name in ("campaigns", "access_groups", "access_users"): if table_name in tables and "settings" in _columns(inspector, table_name): op.drop_column(table_name, "settings") diff --git a/dev/postgres/.env.example b/dev/postgres/.env.example new file mode 100644 index 0000000..0cdc74c --- /dev/null +++ b/dev/postgres/.env.example @@ -0,0 +1,5 @@ +GOVOPLAN_POSTGRES_DB=govoplan +GOVOPLAN_POSTGRES_USER=govoplan +GOVOPLAN_POSTGRES_PASSWORD=govoplan-dev +GOVOPLAN_POSTGRES_PORT=55432 +GOVOPLAN_POSTGRES_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55432/govoplan diff --git a/dev/postgres/README.md b/dev/postgres/README.md new file mode 100644 index 0000000..003a80f --- /dev/null +++ b/dev/postgres/README.md @@ -0,0 +1,126 @@ +# PostgreSQL Development Profile + +GovOPlaN development now defaults to PostgreSQL: + +```text +postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev +``` + +The matching pg-tools URL for `psql`, `pg_dump`, and `pg_restore` is: + +```text +postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev +``` + +Store the password in `~/.pgpass` instead of committing it to a `.env` file. +The devserver and `scripts/launch-dev.sh` honor an explicit `DATABASE_URL` if +you need a different database. + +## Host PostgreSQL Setup + +Create the default local role and database from a host shell: + +```bash +export GOVOPLAN_PG_DB=govoplan_dev +export GOVOPLAN_PG_USER=govoplan_dev +read -rsp "Password for ${GOVOPLAN_PG_USER}: " GOVOPLAN_PG_PASSWORD +echo + +if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${GOVOPLAN_PG_USER}'" | grep -q 1; then + sudo -u postgres createuser --pwprompt "${GOVOPLAN_PG_USER}" +else + sudo -u postgres psql -c "\\password ${GOVOPLAN_PG_USER}" +fi + +if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='${GOVOPLAN_PG_DB}'" | grep -q 1; then + sudo -u postgres createdb -O "${GOVOPLAN_PG_USER}" "${GOVOPLAN_PG_DB}" +fi + +sudo -u postgres psql -d "${GOVOPLAN_PG_DB}" -c "ALTER SCHEMA public OWNER TO ${GOVOPLAN_PG_USER};" + +touch ~/.pgpass +chmod 600 ~/.pgpass +grep -v "^127.0.0.1:5432:${GOVOPLAN_PG_DB}:${GOVOPLAN_PG_USER}:" ~/.pgpass > ~/.pgpass.tmp || true +printf '127.0.0.1:5432:%s:%s:%s\n' "$GOVOPLAN_PG_DB" "$GOVOPLAN_PG_USER" "$GOVOPLAN_PG_PASSWORD" >> ~/.pgpass.tmp +mv ~/.pgpass.tmp ~/.pgpass +chmod 600 ~/.pgpass +``` + +Check access: + +```bash +psql "postgresql://${GOVOPLAN_PG_USER}@127.0.0.1:5432/${GOVOPLAN_PG_DB}" \ + -c 'select current_user, current_database();' +``` + +When running through the VS Codium Flatpak sandbox, host PostgreSQL tools are +available through: + +```bash +flatpak-spawn --host /usr/bin/psql --version +flatpak-spawn --host /usr/bin/pg_dump --version +flatpak-spawn --host /usr/bin/pg_restore --version +``` + +## Run GovOPlaN Against Host PostgreSQL + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m govoplan_core.commands.init_db \ + --database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \ + --with-dev-data + +./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload +``` + +For the full backend and WebUI launcher: + +```bash +cd /mnt/DATA/git/govoplan-core +scripts/launch-dev.sh +``` + +To force the old SQLite fallback for a disposable local run: + +```bash +GOVOPLAN_DEV_DATABASE_BACKEND=sqlite scripts/launch-dev.sh +``` + +## Disposable Docker Testbed + +This testbed is for migration and module-permutation checks. It is not a +production deployment profile. + +```bash +cd /mnt/DATA/git/govoplan-core/dev/postgres +cp .env.example .env +docker compose --env-file .env up -d +``` + +Run the integration check from the core checkout: + +```bash +cd /mnt/DATA/git/govoplan-core +set -a +. dev/postgres/.env +set +a +./.venv/bin/python scripts/postgres-integration-check.py \ + --database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \ + --reset-schema +``` + +`--reset-schema` drops and recreates the `public` schema before every module +set. Use it only against this disposable database. + +Stop the testbed: + +```bash +cd /mnt/DATA/git/govoplan-core/dev/postgres +docker compose --env-file .env down +``` + +Remove all test data: + +```bash +docker compose --env-file .env down -v +``` diff --git a/dev/postgres/docker-compose.yml b/dev/postgres/docker-compose.yml new file mode 100644 index 0000000..b07935c --- /dev/null +++ b/dev/postgres/docker-compose.yml @@ -0,0 +1,22 @@ +services: + postgres: + image: postgres:16-alpine + container_name: govoplan-core-postgres-dev + environment: + POSTGRES_DB: ${GOVOPLAN_POSTGRES_DB:-govoplan} + POSTGRES_USER: ${GOVOPLAN_POSTGRES_USER:-govoplan} + POSTGRES_PASSWORD: ${GOVOPLAN_POSTGRES_PASSWORD:-govoplan-dev} + ports: + - "127.0.0.1:${GOVOPLAN_POSTGRES_PORT:-55432}:5432" + healthcheck: + test: + - CMD-SHELL + - pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}" + interval: 5s + timeout: 3s + retries: 20 + volumes: + - postgres-data:/var/lib/postgresql/data + +volumes: + postgres-data: diff --git a/dev/production-like/.env.example b/dev/production-like/.env.example new file mode 100644 index 0000000..31d05b3 --- /dev/null +++ b/dev/production-like/.env.example @@ -0,0 +1,9 @@ +GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan +GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER=govoplan +GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD=govoplan-dev +GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT=55433 +GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT=56379 + +GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan +GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS=postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan +GOVOPLAN_PRODUCTION_LIKE_REDIS_URL=redis://127.0.0.1:56379/0 diff --git a/dev/production-like/README.md b/dev/production-like/README.md new file mode 100644 index 0000000..b4d12cf --- /dev/null +++ b/dev/production-like/README.md @@ -0,0 +1,49 @@ +# Production-Like Development Profile + +This profile runs the shared services that production depends on while keeping +API, worker, and WebUI code in the editable local repositories. + +It provides: + +- PostgreSQL with a persistent Docker volume +- Redis with append-only persistence +- explicit `ENABLED_MODULES` +- local durable file storage under `runtime/production-like/files` +- a Celery worker process using the same queues as the API + +Start it from the core repository: + +```bash +cd /mnt/DATA/git/govoplan-core +scripts/launch-production-like-dev.sh +``` + +The launcher uses `dev/production-like/.env` when present, otherwise +`dev/production-like/.env.example`. Copy the example when you want local port or +password changes: + +```bash +cp dev/production-like/.env.example dev/production-like/.env +``` + +The API and worker use: + +```text +DATABASE_URL=postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan +REDIS_URL=redis://127.0.0.1:56379/0 +CELERY_ENABLED=true +``` + +Stop the launched API/WebUI/worker with `Ctrl+C`. The PostgreSQL and Redis +containers keep running by default so the next launch is fast. To stop them too: + +```bash +GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh +``` + +To remove all profile data: + +```bash +cd /mnt/DATA/git/govoplan-core/dev/production-like +docker compose --env-file .env.example down -v +``` diff --git a/dev/production-like/docker-compose.yml b/dev/production-like/docker-compose.yml new file mode 100644 index 0000000..63bef16 --- /dev/null +++ b/dev/production-like/docker-compose.yml @@ -0,0 +1,37 @@ +services: + postgres: + image: postgres:16-alpine + container_name: govoplan-production-like-postgres + environment: + POSTGRES_DB: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB:-govoplan} + POSTGRES_USER: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_USER:-govoplan} + POSTGRES_PASSWORD: ${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PASSWORD:-govoplan-dev} + ports: + - "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}:5432" + healthcheck: + test: + - CMD-SHELL + - pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}" + interval: 5s + timeout: 3s + retries: 20 + volumes: + - postgres-data:/var/lib/postgresql/data + + redis: + image: redis:7-alpine + container_name: govoplan-production-like-redis + command: ["redis-server", "--appendonly", "yes"] + ports: + - "127.0.0.1:${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + volumes: + - redis-data:/data + +volumes: + postgres-data: + redis-data: diff --git a/docs/ACCESS_EXTRACTION_PLAN.md b/docs/ACCESS_EXTRACTION_PLAN.md deleted file mode 100644 index dec2ec0..0000000 --- a/docs/ACCESS_EXTRACTION_PLAN.md +++ /dev/null @@ -1,456 +0,0 @@ -# GovOPlaN Access Extraction Plan - -> Backlog state migrated to Gitea issues on 2026-07-06. Keep this document as -> durable extraction context and architecture notes. Track active tasks, -> decisions, blockers, and implementation state in `add-ideas/govoplan-core` -> issues with `module/access` and `source/backlog-import`. - -This plan describes how to extract access, authentication, RBAC, and related -administration behavior from the current compatibility core into a dedicated -`govoplan-access` platform module. - -The goal is not to make access optional in every real deployment. The goal is to -make ownership explicit: the kernel composes modules and exposes contracts; -`govoplan-access` owns identity, sessions, API keys, roles, groups, and access -administration. - -## Current Inventory - -### Existing Access Module Seed - -`govoplan-access` contains the extracted module-shaped seed package under -`src/govoplan_access/backend`. `govoplan-core` keeps compatibility import shims -under `src/govoplan_core/access`: - -- `manifest.py` declares `ModuleManifest(id="access")`. -- `db/models.py` defines access-owned model candidates. -- `auth/principals.py`, `auth/roles.py`, and `auth/tokens.py` contain auth - helper concepts. -- `permissions/definitions.py`, `permissions/evaluator.py`, and - `permissions/registry.py` contain permission catalogue/evaluation concepts. -- `tenancy/datastore.py` contains early tenancy access helpers. - -This seed package is now the starting point for the access module, but it is -not yet the full live source of truth for the running product. - -### Live Compatibility Ownership In Core - -The active implementation still uses core-owned models and services: - -- `src/govoplan_core/db/models.py` - - `Account` - - `Tenant` - - `User` - - `Group` - - `Role` - - `SystemSettings` - - `GovernanceTemplate` - - `GovernanceTemplateAssignment` - - `SystemRoleAssignment` - - `UserGroupMembership` - - `UserRoleAssignment` - - `GroupRoleAssignment` - - `ApiKey` - - `AuthSession` - - `AuditLog` -- `src/govoplan_core/auth/dependencies.py` -- `src/govoplan_core/security/api_keys.py` -- `src/govoplan_core/security/permissions.py` -- `src/govoplan_core/security/sessions.py` -- `src/govoplan_core/security/passwords.py` -- `src/govoplan_core/security/secrets.py` -- `src/govoplan_core/security/module_permissions.py` -- `src/govoplan_core/admin/service.py` -- `src/govoplan_core/admin/governance.py` -- `src/govoplan_core/api/v1/auth.py` -- `src/govoplan_core/api/v1/admin.py` -- `src/govoplan_core/api/v1/admin_schemas.py` -- `src/govoplan_core/api/v1/audit.py` -- `src/govoplan_core/db/bootstrap.py` - -Core still hosts the generic login/settings shell and shared WebUI primitives. -The legacy administration page has moved to the `govoplan-access` WebUI package -and is contributed as the `/admin` route by the access module. Individual admin -panels can now be split further into access, admin, tenancy, policy, and audit -WebUI contributions without changing the core shell route wiring again. - -### Current Module Consumers - -Feature modules no longer import core auth dependency wrappers or access-owned -ORM models. Backend routers import the access-published FastAPI dependency API -from `govoplan_access.backend.auth.dependencies`; runtime cooperation uses -kernel capabilities such as `access.directory`, `campaigns.access`, -`campaigns.mailPolicyContext`, and `campaigns.deliveryTasks`. - -## Target Ownership - -### Kernel - -The kernel keeps only platform composition and stable contracts: - -- module discovery and registry validation -- route aggregation -- migration orchestration -- database engine/session lifecycle -- permission catalogue aggregation -- capability registry -- event/command envelopes -- dependency injection hooks -- health and diagnostics -- compatibility facades while modules migrate - -The kernel should not own account, tenant, role, group, session, or API-key -semantics. - -### `govoplan-access` - -`govoplan-access` owns: - -- accounts -- authentication routes -- session lifecycle -- API keys -- users -- groups and memberships -- roles and assignments -- principal resolution -- permission evaluation -- access administration routes -- access administration WebUI contributions -- access-related migrations -- access permissions and role templates - -### Later Platform Modules - -Some current access-adjacent behavior can remain in access initially, but should -have clean seams for later extraction: - -- `govoplan-tenancy`: tenants, tenant lifecycle, tenant switching, tenant - metadata, tenant settings boundaries. -- `govoplan-policy`: hierarchical policy/effective policy resolution and - provenance display contracts. -- `govoplan-audit`: audit log storage, audit routes, retention hooks, evidence - exports. -- `govoplan-admin`: generic administration shell contributions if they are not - owned by access/tenancy/policy/audit directly. - -## Kernel Contracts To Stabilize First - -Before moving live code, define contracts that feature modules can depend on -without importing access ORM models: - -- `PrincipalResolver` - - resolves the current actor from a request/session/API key. - - returns a stable DTO, not an ORM model. -- `AccessDirectory` - - resolves users, groups, memberships, and display labels by stable IDs. - - supports batch lookups for grids and policy screens. -- `TenantResolver` - - resolves current tenant context and tenant metadata by stable ID. -- `PermissionEvaluator` - - evaluates required scopes for a principal and resource. -- `ResourceAccessProvider` - - lets modules register resource ACL behavior without importing each other. -- `SecretProvider` - - encrypts/decrypts named secrets without tying consumers to access models. -- `AuditSink` - - records audit events without importing audit storage models. - -DTOs should be small and serializable: - -- `PrincipalRef` -- `AccountRef` -- `TenantRef` -- `UserRef` -- `GroupRef` -- `RoleRef` - -## Extraction Stages - -### Stage 0: Baseline Safeguards - -Status: started. - -- Document the kernel/platform module model. -- Add dependency-boundary checks. -- Add backend module permutation startup tests. -- Inventory access/auth/RBAC imports. -- Draft this extraction plan. - -Acceptance criteria: - -- `scripts/check_dependency_boundaries.py` passes. -- backend module permutation tests start every supported module combination. -- current direct imports are tracked as explicit transitional debt. - -### Stage 1: Contract Definitions - -Add kernel-owned protocols and DTOs for access-related interaction. - -Tasks: - -- Add protocol definitions under a kernel contract package. -- Add registry/capability names for access services. -- Keep existing core dependency functions as compatibility wrappers. -- Make wrappers resolve through capabilities when `govoplan-access` is present. -- Add tests for missing-capability behavior and clear error messages. - -Acceptance criteria: - -- Feature modules can receive principal/tenant/group/user references without - importing access ORM models. -- Existing routes continue to work through compatibility wrappers. - -### Stage 2: Create `govoplan-access` - -Create the new repository/package and move the access seed package into it. - -Tasks: - -- Create package metadata and entry points for `govoplan-access`. -- Move `govoplan_core/access` implementation into the new package. -- Publish `ModuleManifest(id="access")`. -- Register access permissions, role templates, routers, and migrations. -- Add compatibility imports in core where needed. -- Add release/dev dependency entries in core. - -Acceptance criteria: - -- `govoplan-core + govoplan-access` starts through normal module discovery. -- `govoplan-core` compatibility imports still work during transition. -- access manifest, migrations, and route contributions are discovered by core. - -### Stage 3: Move Live Models And Migrations - -Move active identity/access models out of `govoplan_core.db.models`. - -Current state: the stable table naming strategy is to keep legacy table names -and move SQLAlchemy class definitions under their platform owners. The old -`govoplan_core.db.models` compatibility re-export has been removed; callers -must import module-owned models or use kernel capabilities. The earlier -`access_*` candidate tables are not active metadata. - -Current table ownership: - -- `govoplan-tenancy`: `tenants` -- `govoplan-access`: `accounts`, tenant memberships in `users`, `groups`, - `roles`, role/group assignment tables, `api_keys`, and `auth_sessions` -- `govoplan-admin`: governance templates and assignments -- `govoplan-audit`: audit log -- `govoplan-core`: system settings - -Tasks: - -- [x] Decide the stable table naming strategy. -- [x] Map legacy model names to access-owned model classes. -- [x] Keep database table names where possible to avoid data migration churn. -- [x] Add Alembic migration metadata owned by the platform module owners. -- [x] Remove core model compatibility aliases after callers moved to - module-owned imports. -- [x] Update bootstrap/create-all compatibility paths. - -Acceptance criteria: - -- Existing development databases migrate without data loss. -- New databases initialize with module-owned metadata. -- Core no longer owns or re-exports live account/user/group/role/session/API-key - model definitions. - -### Stage 4: Move Auth Routes And Dependencies - -Move authentication, sessions, API keys, and principal dependencies. - -Tasks: - -- Move `/api/v1/auth/*` implementation to access. -- Move session/API-key services to access. -- Keep core route compatibility only if required by clients. -- Replace feature-module imports of core auth dependencies with the - access-published FastAPI dependency API. -- Add tests for login, session refresh, tenant selection, API-key auth, and - missing access capability failures. - -Acceptance criteria: - -- Auth behavior works through the access module. -- Feature modules do not import access internals except the published - `govoplan_access.backend.auth.dependencies` dependency API used by FastAPI - routers. -- Core can explain startup failure clearly if auth-required routes are enabled - without the access capability. - -### Stage 5: Move Admin And WebUI Contributions - -Move access administration UI/API ownership into modules. - -Tasks: - -- [x] Move users, groups, roles, API keys, and access settings pages into - `govoplan-access`. -- [ ] Move tenant-specific pages to `govoplan-tenancy` when that module exists, or - keep them temporarily in access with clear boundaries. -- [x] Move overview, system settings, and governance-template panels into - `govoplan-admin`. -- [x] Register admin navigation and admin sections through module - contributions. -- [x] Keep core shell, layout, route rendering, and generic components only. - -Acceptance criteria: - -- Core WebUI shell renders admin/access pages from module contributions. -- Core does not import access page components directly. -- Module nav and route metadata remain serializable. -- The access admin shell consumes module-owned `admin.sections` capability - contributions without importing sibling module panels. - -### Stage 6: Decouple Feature Modules - -Remove direct ORM/model imports from files, mail, and campaign. - -Tasks: - -- Replace `Group`, `Tenant`, `User`, and `UserGroupMembership` imports with - directory/capability lookups. -- Replace direct cross-module cleanup/count queries with registered providers, - events, or module-owned API contracts. -- Replace mail-profile ownership resolution with stable owner references. -- Replace campaign/file access checks with resource ACL provider contracts. - -Acceptance criteria: - -- `govoplan-files`, `govoplan-mail`, and `govoplan-campaign` do not import - access implementation modules or sibling feature modules. -- Dependency-boundary allowlist stays empty after each replacement. - -### Stage 7: Remove Compatibility Debt - -Finalize the split. - -Tasks: - -- Remove obsolete core compatibility aliases. -- Keep the dependency-boundary checker allowlist empty. -- Update release dependency docs. -- Update operator migration notes. -- Add full module permutation tests including access-present and access-absent - behavior. - -Acceptance criteria: - -- Core is a composition kernel. -- Access behavior is owned by `govoplan-access`. -- Feature modules communicate through kernel contracts, capabilities, events, - and their own APIs. - -## Immediate Backlog - -- [x] Document kernel/platform module model. -- [x] Add dependency-boundary check. -- [x] Add backend permutation startup smoke checks. -- [x] Inventory access/auth/RBAC import debt. -- [x] Draft access extraction plan. -- [x] Define kernel access DTOs and protocols. -- [x] Add access capability registry names. -- [x] Register live access directory and tenant resolver capability - implementations backed by the current compatibility tables. -- [x] Register an access tenant-provisioner capability so tenancy can seed - access-owned default roles without importing access service internals. -- [x] Replace `govoplan-files` direct user/group/tenant model imports with the - `access.directory` capability for group membership and share-target checks. -- [x] Replace `govoplan-files` direct campaign model imports with the - `campaigns.access` capability for campaign share/existence checks. -- [x] Replace `govoplan-mail` direct user/group/tenant model imports with - mail-owned policy storage plus access-directory validation and legacy-read - fallback. -- [x] Replace `govoplan-mail` direct campaign model imports with the - `campaigns.mailPolicyContext` capability for campaign-scoped mail policy and - owner context. -- [x] Replace `govoplan-campaign` runtime user/group/tenant model imports with - access-directory lookups and string-based ORM relationships. -- [x] Create `govoplan-access` repository workspace and issue workflow scaffold. -- [x] Create `govoplan-access` repository/package skeleton. -- [x] Move `govoplan_core/access` seed package into `govoplan-access`. -- [x] Add compatibility wrappers in core for old import paths. -- [x] Route existing auth dependency wrappers through access principal/evaluator capabilities. -- [x] Move FastAPI auth dependency wrappers out of core and into - `govoplan-access`. -- [x] Move session, API-key, and password helper services into `govoplan-access`. -- [x] Move interactive auth/session routes behind access module manifest. -- [x] Move legacy admin/API-key routes behind access module manifest. -- [x] Move access-owned legacy admin service helpers into `govoplan-access`. -- [x] Move governance-template CRUD helpers and routes into `govoplan-admin`. -- [x] Move governance-template materialization of access-owned groups and roles - behind the `access.governanceMaterializer` capability. -- [x] Replace legacy access-to-files/campaign admin lookups with module - tenant-summary and group-delete veto providers. -- [x] Split legacy admin route contribution across `govoplan-admin`, - `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit`, and - `govoplan-access` route slices. -- [x] Move legacy admin route-handler ownership out of the access compatibility - router into access, admin, tenancy, policy, and audit module routers. -- [x] Move shared system-settings, tenant-governance, slug/error, and - tenant-count helpers out of access internals into core settings/tenancy - helpers used by the split platform route modules. -- [x] Move campaign schema routes into the campaign route contribution. -- [x] Move development mailbox routes into the mail route contribution. -- [x] Replace core Celery direct campaign/mail imports with the - `campaigns.deliveryTasks` capability. -- [x] Replace core retention direct campaign queries with - `campaigns.policyContext` and `campaigns.retention` capabilities. -- [x] Replace core `create_all` feature-model imports with module registry - metadata discovery. -- [x] Remove transitional boundary checker allowlist entries. -- [x] Move live legacy model definitions out of core and into - `govoplan-access` while preserving existing table names. -- [x] Split the transitional access-owned legacy model graph further into - tenancy, audit, admin, access, and core settings ownership. -- [x] Reverse the tenancy/access dependency direction so `govoplan-access` - depends on `govoplan-tenancy`, and the registry inserts tenancy before access. -- [x] Replace tenancy-to-access default role seeding with an access - tenant-provisioner capability. -- [x] Replace tenancy-to-access owner candidate and owner membership handling - with the access tenant-provisioner capability. -- [x] Replace admin overview counts and audit actor lookup/filtering with the - `access.administration` capability. -- [x] Replace feature-module direct user/group/tenant model imports. - - [x] `govoplan-files` - - [x] `govoplan-mail` - - [x] `govoplan-campaign` -- [x] Replace files/mail/campaign direct cross-module SQL lookups with - provider/capability contracts. -- [x] Move access admin WebUI pages to module route contributions. -- [x] Move generic admin-owned WebUI panels into `govoplan-admin` and register - them through the `admin.sections` UI capability. -- [x] Remove transitional boundary checker allowlist entries as each contract - lands. -- [x] Remove old core import shims for access models, access routes, admin - service helpers, and access-owned security services. -- [ ] Move campaign-scoped mail policy ownership fully behind an API/event - workflow if direct synchronous capability calls become too tight for later - deployment boundaries. - -## Open Decisions - -- Is `govoplan-access` required for any authenticated deployment, while - core-only remains a diagnostics/settings shell? -- Tenant models live in `govoplan-tenancy`; `govoplan-access` depends on - tenancy for authenticated platform composition. -- Historical table names remain stable for painless migrations. -- Should secret encryption remain a kernel primitive or become an access-owned - capability? -- Audit log storage lives in `govoplan-audit`; policy provenance can build on - that module boundary. -- How long should core keep compatibility route paths for existing clients? - -## Verification - -Use the following checks while extracting access: - -```bash -cd /mnt/DATA/git/govoplan-core -./.venv/bin/python scripts/check_dependency_boundaries.py -./.venv/bin/python -m unittest tests.test_module_system -``` - -For each removed dependency-boundary exception, add or update a focused test -that proves the replacement contract starts without the old direct import. diff --git a/docs/ACCESS_RBAC_MODEL.md b/docs/ACCESS_RBAC_MODEL.md new file mode 100644 index 0000000..8d9c1da --- /dev/null +++ b/docs/ACCESS_RBAC_MODEL.md @@ -0,0 +1,288 @@ +# GovOPlaN RBAC And Resource-Access Model + +**Updated:** 2026-07-09 + +## Authorization Equation + +An operation is permitted only when every applicable layer allows it: + +```text +effective role/API-key capability +AND resource ownership/share access +AND workflow state +AND active governance/policy constraints +``` + +RBAC answers what an actor may do. ACLs answer which resource the actor may do +it to. Workflow state and policy decide whether the operation is currently +valid. + +## Identity And Scope + +```text +Account global login identity ++- User membership tenant-local identity + +- direct tenant roles + +- active group memberships + | +- inherited tenant roles + +- tenant-local API keys + +Account ++- direct system-role assignments +``` + +A browser session has one active tenant membership. System privileges do not +silently grant tenant data access. API keys remain tenant-local and receive the +intersection of their configured scopes and their owner's live tenant scopes on +every request. + +## Wildcards + +```text +tenant:* every canonical tenant permission +system:* every canonical system permission +* legacy alias interpreted as tenant:* only +``` + +Tenant wildcards never grant system permissions. + +## Canonical Tenant Permissions + +Campaigns: + +```text +campaign:read +campaign:create +campaign:update +campaign:copy +campaign:archive +campaign:delete +campaign:share +campaign:validate +campaign:build +campaign:review +campaign:send_test +campaign:queue +campaign:control +campaign:send +campaign:retry +campaign:reconcile +``` + +Recipients: + +```text +recipients:read +recipients:write +recipients:import +recipients:export +``` + +Files: + +```text +files:read +files:download +files:upload +files:organize +files:share +files:delete +files:admin +``` + +Reports and audit: + +```text +reports:read +reports:export +reports:send +audit:read +``` + +Mail servers: + +```text +mail_servers:read +mail_servers:use +mail_servers:test +mail_servers:write +mail_servers:manage_credentials +``` + +Tenant administration: + +```text +admin:users:read +admin:users:create +admin:users:update +admin:users:suspend + +admin:groups:read +admin:groups:write +admin:groups:manage_members + +admin:roles:read +admin:roles:write +admin:roles:assign + +admin:api_keys:read +admin:api_keys:create +admin:api_keys:revoke + +admin:settings:read +admin:settings:write +admin:policies:read +admin:policies:write +``` + +## Canonical System Permissions + +```text +system:tenants:read +system:tenants:create +system:tenants:update +system:tenants:suspend + +system:accounts:read +system:accounts:create +system:accounts:update +system:accounts:suspend + +system:roles:read +system:roles:write +system:roles:assign + +system:access:read +system:access:assign + +system:audit:read +system:settings:read +system:settings:write +system:governance:read +system:governance:write +``` + +`system:access:*` remains a read/assignment boundary for cross-tenant and +system access handling. It is not a separate primary UI area. + +## Default Tenant Roles + +- **Owner:** `tenant:*`. At least one active operational owner must remain. +- **Tenant administrator:** settings, policies, users, groups, roles, API keys, + and read access to campaigns/files/reports/audit. Real delivery remains + separately delegable. +- **Administrator:** all tenant permissions for upgraded installations. +- **Access administrator:** membership and assignment management within + delegation limits. +- **Campaign manager:** prepare, validate, and build campaigns; no review + approval or real delivery by default. +- **Reviewer:** inspect and approve prepared campaign messages. +- **Sender:** mock-test, queue, control, send, retry, and reconcile prepared + campaigns; can use/test approved mail profiles. +- **File manager:** managed file operations without campaign delivery rights. +- **Viewer:** read campaigns, recipients, files, and reports. +- **Auditor:** read campaigns, recipient evidence, reports, and audit records; + export detailed evidence. + +## Default System Roles + +- **System owner:** `system:*`, protected. At least one active account must + retain it. +- **System administrator:** all specific system permissions, editable and not + protected. +- **System auditor:** read-only system registry/settings/governance/audit role, + editable. + +## Delegation Ceiling + +For role definition, assignment, and API-key creation: + +```text +requested scopes subset of actor delegateable scopes +``` + +Rules: + +1. Tenant roles may contain tenant scopes only. +2. System roles may contain system scopes only. +3. Definition rights and assignment rights are separate. +4. Group definition and group membership management are separate. +5. API-key scopes are intersected with the owner's current effective scopes on + every request. +6. Suspended accounts, users, tenants, or groups stop contributing access + immediately. +7. Administrative updates are field-sensitive; a user with only status + authority cannot change role assignments. + +## Campaign Ownership And ACLs + +A campaign has exactly one owner: + +```text +owner user OR owner group +``` + +Additional active shares may target users or groups with `read` or `write`. + +Resolution: + +- owner user: read and write; +- member of owner group: read and write; +- explicit read share: read; +- explicit write share: read and write; +- `tenant:*`: tenant-wide ACL bypass; +- ordinary campaign permission without ownership/share: no object access. + +ACLs do not add capabilities. A write share still needs the specific permission +for update, validation, review, send, report, retry, or reconciliation. + +## Files + +The current file access model distinguishes tenant-level file capabilities from +space/folder/file ownership: + +| Scope | Meaning | +| --- | --- | +| `files:read` | browse/read visible file spaces and metadata | +| `files:download` | download file content when ACL permits | +| `files:upload` | create files in writable spaces | +| `files:organize` | create folders, move files, and update metadata where ACL permits | +| `files:share` | share files/spaces according to owner and policy rules | +| `files:delete` | delete or retire files where ACL permits | +| `files:admin` | tenant-wide administration of user/group file spaces | + +External file connections and spaces are additionally constrained by connector +policy and owner/group assignment in the files module. + +## Mail Servers + +| Scope | Meaning | +| --- | --- | +| `mail_servers:read` | profile metadata and effective policy visibility | +| `mail_servers:use` | use an allowed profile for a campaign or message flow | +| `mail_servers:test` | run connectivity tests without revealing secrets | +| `mail_servers:write` | create/update profile metadata where policy allows | +| `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or lower-level credentials where policy allows | + +Reusable encrypted profiles exist. Effective usability is also constrained by +hierarchical mail-profile policy, ownership, allowed/forced profile sets, +credential inheritance mode, lower-level override switches, and allow/deny +patterns. + +## Compatibility Aliases + +Compatibility aliases may exist in backend code for upgraded installations, but +new UI and docs should use canonical scopes. + +Current alias direction: + +```text +* -> tenant:* +system:tenants:write -> create/update/suspend tenant scopes +system:access:write -> system access assignment/write scopes +``` + +A separate `retention:*` family is not currently canonical because retention is +managed through system settings and tenant policy scopes. Add it only if +retention operation duties need separation from general policy/settings +administration. diff --git a/docs/ACTION_EFFECT_AUTOMATION_LAYER.md b/docs/ACTION_EFFECT_AUTOMATION_LAYER.md new file mode 100644 index 0000000..e28166f --- /dev/null +++ b/docs/ACTION_EFFECT_AUTOMATION_LAYER.md @@ -0,0 +1,121 @@ +# Action, Effect, And Automation Layer + +GovOPlaN needs an automation layer because administrative processes will not be +only linear screen flows. Workflows, schedules, imports, connectors, policies, +and external events all need to request governed actions without bypassing the +same safety rules that apply to human users. + +The first implementation should live in `govoplan-workflow` and core contracts. +Create a separate `govoplan-automation` module only if action planning, +schedulers, rule execution, or cross-module automation become too broad for +workflow ownership. + +## Layer Purpose + +The automation layer should provide: + +- a typed action catalogue +- a typed effect catalogue +- consequence preview before execution +- policy and permission checks +- idempotent execution +- audit and provenance records +- retry, quarantine, and manual exception handling +- system-actor execution without hiding responsibility + +Automation is not a shortcut around module boundaries. It is a governed caller +of module capabilities. + +## Action Definition + +An `ActionDefinition` describes something a human or system actor can request. + +Recommended fields: + +- `action_key` +- owning module +- input schema +- actor requirements and required scopes +- required capabilities +- policy checks +- risk level +- reversibility class: reversible, compensatable, corrective-only, or + irreversible +- expected effects +- idempotency key strategy +- audit event names +- preview provider + +Examples: + +- create a case from a form submission +- assign a task +- generate a document from a template +- send a postbox message +- send an email notification +- append evidence to records +- create a payment request +- call an external connector + +## Effect Definition + +An `EffectDefinition` describes the expected and observed result of an action. + +Recommended fields: + +- `effect_key` +- affected module and resource references +- external system references where applicable +- created, changed, deleted, sent, notified, locked, or retained markers +- visibility and privacy classification +- audit event references +- rollback or compensation hints +- operator-facing explanation + +Effects should be recorded even when execution fails partially. This makes +manual recovery and audit review possible. + +## Execution Model + +The runner should execute an action plan as follows: + +1. Resolve actor context: human, delegated actor, or system actor. +2. Validate input schema. +3. Resolve required module capabilities. +4. Run permission and policy checks. +5. Generate a consequence preview. +6. Reserve or verify the idempotency key. +7. Execute the owning module capability. +8. Record observed effects. +9. Emit events and audit records. +10. Mark the command complete, retryable, quarantined, or requiring manual + intervention. + +The runner must never advance workflow state past a required side effect unless +the action definition explicitly allows asynchronous completion and the pending +state is visible. + +## Failure States + +Automation should use explicit failure states: + +- `blocked`: policy, permission, missing capability, or invalid input prevents + execution. +- `retryable`: transient transport, timeout, rate-limit, or lock conflict. +- `quarantined`: unexpected response, schema mismatch, unsafe partial result, + or unknown external state. +- `manual_required`: human decision or correction is needed. +- `compensation_required`: a later action must correct an already observed + side effect. + +These states should be visible in workflow, task, and admin diagnostics. + +## Boundary + +Core may own stable DTOs, registry contracts, and generic audit/event hooks. +`govoplan-workflow` should own the first runner because workflow is the first +module that coordinates cross-module process actions. + +Domain modules own their own action providers. For example, templates own +document generation actions, postbox owns postbox message actions, and +connectors own external handoff actions. diff --git a/docs/CATALOG_TRUST_AND_LICENSING.md b/docs/CATALOG_TRUST_AND_LICENSING.md deleted file mode 100644 index f4422ea..0000000 --- a/docs/CATALOG_TRUST_AND_LICENSING.md +++ /dev/null @@ -1,185 +0,0 @@ -# Module Catalog Trust And Licensing - -GovOPlaN module install and uninstall must remain operator-controlled. The -running server may plan and validate package changes, but package mutation is -performed by the separate installer daemon or an operator shell during -maintenance mode. - -## Roles - -`govoplan-web` is the public static distribution surface for official catalog -resources: - -- signed module package catalogs, grouped by release channel -- public catalog keyrings -- public license verification keyrings -- examples and operator-facing download paths - -`govoplan-core` is the verifier and orchestrator: - -- fetches a local or remote module catalog -- verifies catalog signatures against configured trusted keys -- enforces approved release channels -- rejects expired or not-yet-valid catalogs -- records accepted catalog sequence numbers for replay protection -- checks catalog entry license feature requirements before planning installs -- writes installer plans and request records - -Feature and platform modules own their package artifacts, manifests, migration -metadata, retirement providers, and optional lifecycle behavior. - -## Catalog Source - -Core accepts either a local catalog file or a remote URL: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG=/srv/govoplan/catalogs/stable.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json -``` - -If both file and URL are set, the URL wins. The cache is used when a remote -fetch fails, so an operator can still inspect the last known catalog. A cached -catalog must still pass signature, freshness, channel, and replay validation. - -## Catalog Shape - -An official catalog is a JSON object with: - -- `catalog_version` -- `channel` -- `sequence` -- `generated_at` -- `not_before` when delayed activation is needed -- `expires_at` -- `modules` -- `signatures` - -Each module entry can declare: - -- backend package name and pinned install reference -- WebUI package name and pinned install reference -- display metadata and tags -- `license_features`, the feature entitlements required to plan that install - -The signature is Ed25519 over canonical JSON with both `signature` and -`signatures` removed. Core accepts the legacy single `signature` field and the -new `signatures` array. - -## Keyrings And Rotation - -Trusted catalog keys are configured locally: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":""}' -``` - -For development or tightly controlled deployments, a keyring can be read from a -URL and cached: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL=https://govoplan.example/catalogs/v1/keyring.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE=/srv/govoplan/runtime/catalog-cache/keyring.json -``` - -Production installations should pin the trusted keyring locally or ship it -through deployment configuration. Fetching trusted keys from the same public -origin as the catalog is convenient, but that origin must not become the only -trust root. - -Keyring entries support: - -- `key_id` -- `public_key` or `public_key_base64` -- `status`: `active`, `next`, `retired`, `revoked`, or `disabled` -- `not_before` -- `not_after` - -Rotation process: - -1. Add the next public key to the local trusted keyring with status `next`. -2. Publish catalogs signed by both current and next keys. -3. Upgrade installations so the next key is locally trusted. -4. Promote the next key to `active`. -5. Retire the old key only after every supported installation trusts the new - key. -6. Mark a compromised key `revoked` and publish a higher sequence catalog - signed by an uncompromised key. - -## Replay And Freshness - -Use replay state in production: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true -``` - -Core records the accepted sequence per channel after a catalog entry is planned -from the admin interface. With strict sequence enforcement, a previously -accepted sequence is rejected; without strict enforcement, only older sequences -are rejected. - -Catalogs should always expire. Long-lived catalogs make rollback and key -compromise harder to reason about. - -## Release Channels - -Approved channels are deployment policy: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts -GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true -``` - -The admin UI can display other catalog metadata, but core rejects catalogs from -unapproved channels when validation is configured. - -## Licensing - -Catalog entries can require license features: - -```json -"license_features": ["module.mail", "support.standard"] -``` - -Core checks those requirements against an offline license file before allowing -the entry into the install plan. - -```bash -GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json -GOVOPLAN_LICENSE_ENFORCEMENT=true -GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json -``` - -License files are JSON objects with: - -- `license_id` -- `subject` -- `features` -- `valid_from` -- `valid_until` -- `signature` - -License enforcement can run in observe-only mode by leaving -`GOVOPLAN_LICENSE_ENFORCEMENT` unset. In that mode, missing or invalid license -data is surfaced as a warning but does not block planning. - -Licensing is intentionally separate from open-source code licensing. The -catalog/license mechanism can govern support channels, official release -eligibility, hosted update access, professional support, or commercial -entitlements without changing the source license of the repositories. - -## Production Gaps - -The current implementation validates signed catalog metadata and offline -license entitlements. Production-grade distribution still needs: - -- artifact digest verification for package archives or registry artifacts -- SBOM/provenance fields and verification workflow -- hardened catalog publishing pipeline in `govoplan-web` -- automated key rotation runbook and emergency revocation procedure -- license issuance and renewal tooling -- tests for remote catalog cache fallback and replay-state upgrades - diff --git a/docs/CODEX_WORKFLOW.md b/docs/CODEX_WORKFLOW.md index 46c7dc2..39c0f46 100644 --- a/docs/CODEX_WORKFLOW.md +++ b/docs/CODEX_WORKFLOW.md @@ -27,6 +27,12 @@ trust_level = "trusted" [projects."/mnt/DATA/git/govoplan-access"] trust_level = "trusted" +[projects."/mnt/DATA/git/govoplan-organizations"] +trust_level = "trusted" + +[projects."/mnt/DATA/git/govoplan-identity"] +trust_level = "trusted" + [projects."/mnt/DATA/git/govoplan-mail"] trust_level = "trusted" diff --git a/docs/CONFIGURATION_PACKAGES.md b/docs/CONFIGURATION_PACKAGES.md index d2fb858..c4e1407 100644 --- a/docs/CONFIGURATION_PACKAGES.md +++ b/docs/CONFIGURATION_PACKAGES.md @@ -9,6 +9,13 @@ Example: an application-handling package could configure a public portal form, a case workflow, task creation, a mail template, payment processing, access roles, audit evidence, and the interface bindings between those modules. +The guiding reference scenario is the government-operations permit journey in +`docs/GOVOPLAN_MASTER_ROADMAP.md`: a person applies through the portal, +uploads files, receives workflow-driven messages and appointment proposals, has +a case opened, gets a permit generated from a template, and completes payment. +Configuration packages are the mechanism that should make such processes +reusable and safely importable. + This document is durable architecture context and should be mirrored to the Gitea wiki. Active implementation should be tracked in Gitea issues. @@ -97,6 +104,33 @@ export(selection, context) -> fragment, data_placeholders, warnings health(import_result, context) -> diagnostics ``` +The initial core contract lives in +`govoplan_core.core.configuration_packages`. Modules register providers through +module-specific capability keys and implement the `ConfigurationProvider` +protocol. The generic `configuration.provider` key names the contract and can be +used in package requirements; concrete providers such as `access.configuration` +are resolved by the admin package wizard. The first DTO surface includes package +manifests, module/capability requirements, module-owned fragments, diagnostics, +required operator data, dry-run plan items, apply results, export selections, +and export results. + +The initial implementation includes provider-neutral orchestration helpers: + +- `dry_run_configuration_package(...)` +- `apply_configuration_package(...)` +- `export_configuration_package(...)` + +The first concrete provider is `govoplan_access.backend.configuration_provider`. +It supports access-owned `roles`, `groups`, and `group_role_assignments` +fragments and applies them idempotently. + +The admin wizard backend starts with these routes: + +- `GET /api/v1/admin/configuration-packages/catalog` +- `POST /api/v1/admin/configuration-packages/dry-run` +- `POST /api/v1/admin/configuration-packages/apply` +- `POST /api/v1/admin/configuration-packages/export` + ## Import Flow 1. Select a package from a trusted catalog or upload a package file. @@ -168,6 +202,20 @@ Configuration catalogs should follow the existing module package catalog model: a file-backed or remotely fetched JSON catalog with Ed25519 signatures, channel gating, trusted key ids, and operator-controlled trust policy. +The initial catalog validator mirrors the module catalog environment model with +configuration-specific names: + +```bash +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG=/srv/govoplan/configuration-catalogs/stable.json +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_URL=https://govoplan.example/configuration-catalogs/stable.json +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/configuration-catalog-cache/stable.json +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/configuration-catalog-keyring.json +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/configuration-catalog-sequences.json +GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true +``` + Catalog entries should include: - package id, version, name, description, publisher, tags, and channel diff --git a/docs/DEPENDENCY_AUDITS.md b/docs/DEPENDENCY_AUDITS.md new file mode 100644 index 0000000..98946f1 --- /dev/null +++ b/docs/DEPENDENCY_AUDITS.md @@ -0,0 +1,67 @@ +# Dependency Audits + +GovOPlaN keeps dependency vulnerability checks reproducible but separate from +the fast local smoke suite, because both Python and npm audits need network +metadata and can fail for newly disclosed advisories without a source change. + +## Local Workflow + +Install the development audit dependency once: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m pip install -r requirements-dev.txt +``` + +Run both backend and WebUI production audits: + +```bash +cd /mnt/DATA/git/govoplan-core +bash scripts/check-dependency-audits.sh +``` + +The script runs: + +- `scripts/check-dependency-hygiene.sh` for pip resolver consistency, stale + legacy editable package metadata, deprecated framework constants, and the + Starlette `TestClient` deprecation smoke when test dependencies are present +- `python -m pip_audit --progress-spinner off` +- `npm audit --omit=dev` in `webui` + +For fast local checks without vulnerability metadata lookups, run: + +```bash +cd /mnt/DATA/git/govoplan-core +CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh +``` + +This is also part of `scripts/check-focused.sh`, so resolver drift and +deprecation regressions fail close to the code change that introduced them. + +Override tool paths when testing from a disposable environment: + +```bash +PYTHON=/tmp/govoplan-audit/bin/python \ +NPM=/home/zemion/.nvm/versions/node/v22.22.3/bin/npm \ +bash scripts/check-dependency-audits.sh +``` + +## CI Workflow + +`.gitea/workflows/dependency-audit.yml` installs release dependencies from +tagged package refs, installs `pip-audit`, and runs the same script on pushes, +pull requests, and a weekly schedule. + +The workflow intentionally uses release dependency refs instead of local +`file:` or editable sibling paths. Development lockfiles may keep local module +links, but release audit results should represent the installable product. + +## Recording Results + +When closing or triaging dependency-audit issues, add a short dated note under +`docs/audits/`. Record: + +- the commands that were run +- whether Python and npm passed +- any advisories accepted as temporary risk +- follow-up issue links for required upgrades diff --git a/docs/DEPLOYMENT_OPERATOR_GUIDE.md b/docs/DEPLOYMENT_OPERATOR_GUIDE.md new file mode 100644 index 0000000..f3e2a2b --- /dev/null +++ b/docs/DEPLOYMENT_OPERATOR_GUIDE.md @@ -0,0 +1,390 @@ +# GovOPlaN Deployment Operator Guide + +This guide defines the current install/runtime configuration contract and the +operator flow for a production-realistic self-hosted deployment. Keep secrets in +the deployment environment or a secret manager; do not commit populated `.env` +files. + +## Runtime Configuration Contract + +### Required Runtime Identity + +| Setting | Required outside dev | Purpose | +| --- | --- | --- | +| `APP_ENV` | yes | Runtime profile. Use `prod`, `staging`, or a deployment-specific value outside local development. | +| `MASTER_KEY_B64` | yes | Fernet key or base64 encoded 32-byte key used for encrypted module secrets. Rotate through an explicit operator plan. | +| `DATABASE_URL` | yes | SQLAlchemy database URL for core and installed modules. SQLite is supported for dev/small installs; PostgreSQL is the preferred production target. | +| `ENABLED_MODULES` | yes | Comma-separated startup module set. Keep `tenancy,access` enabled; keep `admin` enabled for operator UI. | + +Generate a local key for a new non-production environment: + +```bash +python - <<'PY' +from cryptography.fernet import Fernet +print(Fernet.generate_key().decode()) +PY +``` + +### Database And Migrations + +| Setting | Default | Notes | +| --- | --- | --- | +| `DATABASE_URL` | `postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev` | Local development and production-like profiles use PostgreSQL. Use `GOVOPLAN_DEV_DATABASE_BACKEND=sqlite` only for disposable SQLite runs. | +| `DEV_AUTO_MIGRATE_ENABLED` | `true` | Dev convenience only. Production should run migration commands explicitly during deployment. | +| `DEV_BOOTSTRAP_ENABLED` | `false` | Dev bootstrap only. `govoplan_core.devserver` and `scripts/launch-dev.sh` default it to `true`; use controlled first-admin creation outside dev. | + +Operator rule: take a database backup before applying migrations or destructive +module retirement. For non-SQLite databases, configure deployment-specific +backup/restore hooks for the module installer. + +### PostgreSQL Production Target + +PostgreSQL is the primary development and production target. SQLite remains +supported only for tiny disposable profiles and unit-test style smoke runs. +Production/staging deployments should use a managed PostgreSQL database and +explicit migration commands. + +Install the server extra so the `psycopg` driver is available: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m pip install -r requirements-release.txt +``` + +Example runtime database URLs: + +```bash +export DATABASE_URL='postgresql+psycopg://govoplan:change-me@db.example.internal:5432/govoplan' +export GOVOPLAN_DATABASE_URL_PGTOOLS='postgresql://govoplan:change-me@db.example.internal:5432/govoplan' +``` + +Use the SQLAlchemy URL for GovOPlaN. Use the pg-tools URL for `pg_dump`, +`pg_restore`, and `psql`; these tools do not understand the +`postgresql+psycopg://` driver marker. + +Bootstrap or upgrade the schema explicitly during deployment: + +```bash +export APP_ENV=prod +export ENABLED_MODULES=tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops +./.venv/bin/python -m govoplan_core.commands.init_db \ + --database-url "$DATABASE_URL" +``` + +Backup and restore-check before migration-bearing package changes: + +```bash +pg_dump --format=custom \ + --file "$PWD/runtime/govoplan-$(date +%Y%m%d%H%M%S).dump" \ + "$GOVOPLAN_DATABASE_URL_PGTOOLS" +pg_restore --list "$PWD/runtime/govoplan-YYYYMMDDHHMMSS.dump" >/dev/null +``` + +Restore a checked backup to the target database: + +```bash +pg_restore --clean --if-exists \ + --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" \ + "$PWD/runtime/govoplan-YYYYMMDDHHMMSS.dump" +``` + +For local development, create the host database described in +`dev/postgres/README.md`, then run: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python -m govoplan_core.commands.init_db \ + --database-url postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev \ + --with-dev-data +./.venv/bin/python -m govoplan_core.devserver --smoke --no-reload +scripts/launch-dev.sh +``` + +For disposable local validation against a throwaway PostgreSQL instance, use +the bundled PostgreSQL testbed: + +```bash +cd /mnt/DATA/git/govoplan-core/dev/postgres +cp .env.example .env +docker compose --env-file .env up -d + +cd /mnt/DATA/git/govoplan-core +set -a +. dev/postgres/.env +set +a +./.venv/bin/python scripts/postgres-integration-check.py \ + --database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \ + --reset-schema +``` + +The integration check runs migrations and startup smoke checks across the +standard module permutations. `--reset-schema` is destructive and belongs only +on throwaway databases. + +### Broker And Workers + +| Setting | Default | Notes | +| --- | --- | --- | +| `REDIS_URL` | `redis://redis:6379/0` | Celery broker/result backend when async workers are enabled. | +| `CELERY_ENABLED` | `false` | Local/dev can send synchronously. Production campaign delivery should run workers and set this to `true`. | +| `CELERY_QUEUES` | `send_email,append_sent,default` | Queue list expected by worker/process manager definitions. | + +Worker command: + +```bash +python -m celery -A govoplan_core.celery_app:celery worker \ + --queues send_email,append_sent,default \ + --loglevel INFO +``` + +### Storage + +| Setting | Default | Notes | +| --- | --- | --- | +| `FILE_STORAGE_BACKEND` | `local` | Use `local` for dev/small deployments; use object storage when files must scale independently. | +| `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Must live on durable storage and be backed up when `FILE_STORAGE_BACKEND=local`. | +| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Read-only fallback roots for migrated local files. | +| `FILE_STORAGE_S3_ENDPOINT_URL` | empty | Object-store endpoint for the files module. | +| `FILE_STORAGE_S3_REGION` | empty | Object-store region. | +| `FILE_STORAGE_S3_ACCESS_KEY_ID` | empty | Secret; inject through deployment environment. | +| `FILE_STORAGE_S3_SECRET_ACCESS_KEY` | empty | Secret; inject through deployment environment. | +| `FILE_STORAGE_S3_BUCKET` | `files` | Managed-file object bucket. | + +Legacy `S3_*` settings remain for older storage paths but new deployments should +prefer `FILE_STORAGE_*`. + +### HTTP, Cookies, And Base URLs + +| Setting | Default | Notes | +| --- | --- | --- | +| `CORS_ORIGINS` | local dev origins | Set to the exact WebUI origins in staging/production. | +| `AUTH_SESSION_COOKIE_NAME` | configured default | Change only through a controlled rollout because it logs users out. | +| `AUTH_CSRF_COOKIE_NAME` | configured default | Must match WebUI/API deployment. | +| `AUTH_COOKIE_SECURE` | `false` | Set `true` behind HTTPS. | +| `AUTH_COOKIE_SAMESITE` | `lax` | Use a stricter value only after testing login and CSRF flows. | +| `AUTH_COOKIE_DOMAIN` | empty | Set only when the API and WebUI intentionally share a parent domain. | + +Public URLs are currently supplied by deployment/reverse-proxy configuration and +module settings. Do not hardcode them in core; configuration packages should ask +for portal, WebUI, postbox, and notification URLs when they become relevant. + +### Module Catalogs, Licenses, And Trust Roots + +| Setting | Purpose | +| --- | --- | +| `GOVOPLAN_MODULE_PACKAGE_CATALOG_URL` or `GOVOPLAN_MODULE_PACKAGE_CATALOG` | Module package catalog source. | +| `GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE` | Preferred production keyring path. | +| `GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL` | Approved catalog channel, for example `stable`. | +| `GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE` | Trusted license issuer keyring path. | +| `GOVOPLAN_LICENSE_ENFORCEMENT` | Enables license enforcement when set to `true`. | + +Trust roots are deployment-managed and should not be editable through the +running WebUI. + +### Mail Test Credentials + +Dedicated SMTP/IMAP test credentials belong to the mail/campaign test-bed +configuration, not the core runtime contract. Store them in a local ignored +`.env` file for the test bed or in CI secrets. Required values are: + +- SMTP host, port, TLS mode, username, password, and envelope/from address. +- IMAP host, port, TLS mode, username, password, and append folder. +- At least one recipient mailbox that is safe for automated send tests. + +## First Deployment Flow + +1. Create an environment file or secret set with the runtime contract above. +2. Install the tagged core and module packages from `requirements-release.txt`. +3. Build the WebUI from `webui/package.release.json` or deploy a prebuilt + artifact from the same release tag. +4. Run database migrations with the target `DATABASE_URL`. +5. Create the first tenant and system owner through the controlled bootstrap or + one-time admin command for the deployment. +6. Start the API service with `govoplan_core.server.app:app`. +7. Start workers when `CELERY_ENABLED=true`. +8. Start the WebUI/reverse proxy and verify CORS/cookie settings. +9. Open Admin > System > Modules, verify enabled modules, and save desired + module state if it differs from `ENABLED_MODULES`. +10. Run health checks: + +```bash +curl -fsS http://127.0.0.1:8000/health +curl -fsS http://127.0.0.1:8000/api/v1/platform/modules +``` + +Authenticated health details require `system:settings:read`: + +```bash +curl -fsS -H "X-API-Key: $GOVOPLAN_HEALTH_API_KEY" \ + http://127.0.0.1:8000/health/details +``` + +## Production-Like Dev Profile + +Use this profile to verify deployment behavior without publishing packages or +using real production credentials. The canonical launcher keeps API, worker, and +WebUI code in the editable repositories while Docker provides PostgreSQL and +Redis: + +```bash +cd /mnt/DATA/git/govoplan-core +scripts/launch-production-like-dev.sh +``` + +The launcher uses `dev/production-like/.env` when present, otherwise the checked +in `.env.example`. It runs: + +- PostgreSQL on `127.0.0.1:55433` +- Redis on `127.0.0.1:56379` +- explicit `ENABLED_MODULES` +- explicit migrations and `--with-dev-data` bootstrap +- API via the module-aware devserver +- a Celery worker for `send_email,append_sent,default` +- WebUI through the Vite dev server +- durable local files under `runtime/production-like/files` + +This profile validates explicit migration execution, config loading, module +discovery, route aggregation, local storage paths, Redis broker connectivity, +worker heartbeats, and health/readiness startup without real production +credentials. It does not replace a managed PostgreSQL/Redis/WebUI/worker +deployment test. + +To stop PostgreSQL and Redis when the launcher exits: + +```bash +GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT=1 scripts/launch-production-like-dev.sh +``` + +## Module Install/Uninstall Operations + +Use Admin > System > Modules for planning. The running API server validates and +queues install plans; it does not run pip/npm or restart itself from an HTTP +request. Package mutation belongs to the trusted installer CLI/daemon in an +operator shell while maintenance mode is active. + +Preflight from the server shell: + +```bash +govoplan-module-installer --format shell +``` + +Apply a prepared plan directly from a controlled shell: + +```bash +govoplan-module-installer --apply --build-webui +``` + +For production-like runs, prefer supervised mode with migrations, database +backup/restore hooks, restart commands, and health checks: + +```bash +govoplan-module-installer \ + --supervise \ + --migrate \ + --restart-command 'systemctl restart govoplan-api' \ + --restart-command 'systemctl restart govoplan-worker' \ + --health-url http://127.0.0.1:8000/health \ + --database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL_PGTOOLS" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \ + --database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" "$GOVOPLAN_DATABASE_BACKUP_PATH"' +``` + +To let the admin UI submit work without executing package managers inside the +API process, run the daemon in a separate operator shell: + +```bash +govoplan-module-installer \ + --daemon \ + --migrate \ + --build-webui \ + --database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL_PGTOOLS" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \ + --database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL_PGTOOLS" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --health-url http://127.0.0.1:8000/health \ + --restart-command '' +``` + +The daemon claims one queued request at a time and writes request/run records +below `runtime/module-installer`. For process-manager one-shot usage or tests, +use `--daemon-once`. Check daemon status with: + +```bash +govoplan-module-installer --daemon-status --format json +``` + +The installer uses a runtime lock, snapshots `pip freeze` plus WebUI package +files, writes a run record, and marks planned rows as applied only after all +commands succeed. With `--migrate`, SQLite databases are backed up through +SQLite's backup API; non-SQLite databases require +`--database-backup-command`, `--database-restore-check-command`, and +`--database-restore-command`. + +Database hook commands receive: + +- `GOVOPLAN_INSTALLER_RUN_DIR` +- `GOVOPLAN_DATABASE_URL` +- `GOVOPLAN_DATABASE_URL_PGTOOLS` for PostgreSQL tools +- `GOVOPLAN_DATABASE_BACKUP_PATH` +- `GOVOPLAN_DATABASE_BACKUP_METADATA` + +Avoid embedding secrets directly in commands; prefer environment variables, +service credentials, or deployment-local secret injection. + +Inspect installer history and lock state from the operator shell: + +```bash +govoplan-module-installer --list-runs --format json +govoplan-module-installer --show-run --format json +govoplan-module-installer --lock-status --format json +govoplan-module-installer --list-requests --format json +govoplan-module-installer --show-request --format json +govoplan-module-installer --cancel-request --format json +govoplan-module-installer --retry-request --format json +``` + +Rollback uses the saved run snapshot: + +```bash +govoplan-module-installer --rollback +govoplan-module-installer --rollback --database-restore-command '' +``` + +Uninstall is non-destructive by default. A planned uninstall row can set +`destroy_data: true` to request destructive module retirement. The module must +provide an automated retirement provider, and the installer snapshots the +database before dropping module-owned tables. + +Run the rollback drill before relying on installer automation in a new +environment: + +```bash +./.venv/bin/python scripts/module-installer-rollback-drill.py --format json +``` + +The drill uses temporary SQLite databases and simulated package commands. It +does not install or uninstall real packages. It exercises: + +- package command failure followed by supervised rollback; +- migration failure with a SQLite database snapshot; +- restart-command failure; +- health timeout after restart; +- destructive retirement executor failure with database rollback; +- PostgreSQL-style backup, restore-check, and restore hooks; +- daemon heartbeat, request queue claim/update, retry/cancel, and stale lock + detection/removal. + +See `RELEASE_DEPENDENCIES.md` for release package refs, migration baseline +checks, catalog trust, signing, keyring, replay, and license operation. + +## Operator Checklist + +- Runtime secrets are injected outside git. +- `MASTER_KEY_B64` is set and backed up securely. +- Database backup and restore commands are tested. +- File/object storage is durable and backed up. +- `CORS_ORIGINS` and cookie settings match the deployed WebUI origin. +- Redis and workers are running before `CELERY_ENABLED=true`. +- Module catalog and license keyrings are pinned locally. +- Health endpoints are monitored. +- Test SMTP/IMAP credentials are non-production and isolated. +- Module installer rollback drill has passed in the deployment environment. diff --git a/docs/DOCUMENTATION_MAP.md b/docs/DOCUMENTATION_MAP.md new file mode 100644 index 0000000..964dc73 --- /dev/null +++ b/docs/DOCUMENTATION_MAP.md @@ -0,0 +1,51 @@ +# GovOPlaN Documentation Map + +This map defines the source-of-truth documents for the current repository docs. +Use it to avoid duplicating long procedures across architecture, release, +operator, and roadmap pages. + +## Core Platform + +| Topic | Canonical document | Notes | +| --- | --- | --- | +| Module architecture and kernel contracts | `MODULE_ARCHITECTURE.md` | Stable module contracts, API efficiency contracts, durable boundary decisions, lifecycle, and WebUI contribution rules. | +| RBAC and resource access | `ACCESS_RBAC_MODEL.md` | Current permission, role, API-key, and resource-access model. | +| Governance hierarchy | `GOVERNANCE_MODEL.md` | System, tenant, user/group, campaign policy inheritance and admin UI structure. | +| Policy decision DTOs and provenance | `POLICY_CONTRACTS.md` | Shared explain/provenance shape; module-specific policy docs should link here. | +| Events and audit trace context | `EVENTS_AND_AUDIT.md` | Event dispatch semantics and audit payload conventions. | +| Action/effect automation layer | `ACTION_EFFECT_AUTOMATION_LAYER.md` | Action/effect contracts, consequence preview, runner semantics, and module boundary for automation. | +| Postbox E2EE target architecture | `POSTBOX_E2EE_ARCHITECTURE.md` | Strategic encrypted postbox/mailbox model, key ownership, role mailbox semantics, and retraction limits. | + +## Release And Operations + +| Topic | Canonical document | Notes | +| --- | --- | --- | +| Runtime configuration and operator flow | `DEPLOYMENT_OPERATOR_GUIDE.md` | Production/staging configuration, migrations, backups, installer operation, and rollback drill. | +| Release dependencies and catalogs | `RELEASE_DEPENDENCIES.md` | Release package refs, migration baselines, release lockfiles, catalog trust/licensing, catalog publishing, and release checklist. | +| Dependency vulnerability audits | `DEPENDENCY_AUDITS.md` | Local and CI audit commands plus dated audit result notes. | +| Remote WebUI bundle design | `REMOTE_WEBUI_BUNDLES.md` | Experimental controlled-deployment design; normal releases still use package builds. | + +## Product And Module Planning + +| Topic | Canonical document | Notes | +| --- | --- | --- | +| Product roadmap and module routing | `GOVOPLAN_MASTER_ROADMAP.md` | Product-level sequencing, implementation gates, issue routing, and missing-module decisions. | +| UI/UX decisions | `UI_UX_DECISION_LEDGER.md` | Binding guided-UI decisions, open decisions, impact index, and review checklist. | +| Interface ethics and design doctrine | `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` | Product-level doctrine for context, decision, consequence, contestability, responsibility, and traceability. | +| Public-sector integration posture | `PUBLIC_SECTOR_INTEGRATION_STRATEGY.md` | Strategy index; executable target inventory lives in `govoplan-connectors`. | +| Configuration packages | `CONFIGURATION_PACKAGES.md` | Package model, provider contract, import/export flow, and tracking slices. | + +## Workflow Docs + +| Topic | Canonical document | Notes | +| --- | --- | --- | +| Gitea issues and wiki sync | `GITEA_ISSUES.md` | Issue labels, imports, wiki mirroring, and Codex state updates. | +| Codex local workflow | `CODEX_WORKFLOW.md` | Local agent setup and focused verification commands. | + +## Cross-Repo Rule + +Core docs may keep strategy, kernel contracts, and routing decisions. Module +repositories should own executable module behavior, concrete API/UI contracts, +and operator notes for their own module. When content spans both, keep the +durable decision in core and link to the module document for implementation +details. diff --git a/docs/EVENTS_AND_AUDIT.md b/docs/EVENTS_AND_AUDIT.md new file mode 100644 index 0000000..16f9cfe --- /dev/null +++ b/docs/EVENTS_AND_AUDIT.md @@ -0,0 +1,180 @@ +# Events And Audit + +GovOPlaN uses a small kernel event contract first, not a broad command bus. +Commands remain module-owned application-service methods or API endpoints until +there is a concrete need for durable asynchronous command orchestration. Events +are facts about completed work and are safe for audit, projections, optional +module reactions, and operator diagnostics. + +## Production Transport Decision + +The first production target is a **database outbox plus in-process immediate +dispatch**: + +- Use `govoplan_core.core.events.PlatformEvent` for domain and platform events. +- Use `EventBus` as the in-process dispatch contract for same-process module + reactions that are safe to run inline. +- Persist durable integration/workflow events through a database outbox before + acknowledging the state change that produced them. +- Drain the outbox through a small dispatcher process. The dispatcher may call + in-process handlers in the same deployment first, but its storage contract is + database-backed. +- Treat Redis/Celery as worker/job infrastructure, not as the authoritative + first event transport. A Celery dispatcher can consume the outbox later. +- Keep the dispatch implementation pluggable behind the `PlatformEvent` + envelope so a future message broker can be added without changing event + producers. +- Keep commands out of the kernel until workflows need retryable, durable, + operator-visible command records. + +This keeps the first contract small, PostgreSQL-friendly, auditable, and +recoverable after process crashes. It also avoids making Redis a correctness +dependency for deployments that only need synchronous mail/tests or light +background work. + +## Dispatch Semantics + +Event producers should write their domain state and outbox event in the same +database transaction wherever possible. Handlers must be idempotent because the +outbox dispatcher can retry after a crash or timeout. + +Recommended first outbox columns: + +- `event_id`, `event_type`, `module_id` +- `correlation_id`, `causation_id` +- `payload`, `occurred_at` +- `available_at`, `attempt_count`, `claimed_at`, `claim_token` +- `processed_at`, `last_error` + +Inline `EventBus` handlers are allowed only for non-critical local reactions. +Anything that must survive process failure, restart, package update, or worker +redeployment belongs in the outbox. + +## Trace IDs + +Every `PlatformEvent` has: + +- `event_id`: unique ID for that event. +- `correlation_id`: stable ID for the whole request, workflow, or job. +- `causation_id`: the event ID or external operation ID that caused this + event. + +The FastAPI app factory creates an event context for every request. It accepts +`X-Correlation-ID` or `X-Request-ID` when the value is a compact safe trace ID, +otherwise it generates a new ID. Responses include `X-Correlation-ID`. + +Audit logging reads the current event context and stores trace IDs in +`details._trace`. Callers can also pass explicit `correlation_id` and +`causation_id` to `audit_event` or `audit_from_principal`. + +Admin and lifecycle code should use the compact operational detail shape +documented in `govoplan-audit/docs/AUDIT_TRACE_CONTEXT.md`. The core +`audit_operation_context` helper preserves `module_id`, `request_id`, `run_id`, +`outcome`, and `_trace` while applying the shared audit redaction pass to +additional detail values. + +## Audit MVP Boundary + +`govoplan-audit` owns: + +- the `audit_log` table and audit API routes +- audit route contribution through its module manifest +- audit retention behavior in cooperation with policy/retention settings +- future audit sink/export capability implementations + +`govoplan-core` owns: + +- `AuditEvent` and `AuditSink` protocol contracts +- request and event trace context +- the compatibility `audit_event` helper while routes are still migrating +- retention orchestration that calls module capabilities + +Feature modules should record audit facts through a small audit API or future +`audit.sink` capability. They should not import audit storage internals. + +## Initial Domain Event Inventory + +Access: + +- `access.account.created` +- `access.account.updated` +- `access.membership.created` +- `access.membership.updated` +- `access.group.created` +- `access.group.updated` +- `access.role.created` +- `access.role.updated` +- `access.role.deleted` +- `access.api_key.created` +- `access.api_key.revoked` +- `access.session.created` +- `access.session.revoked` + +Tenancy: + +- `tenancy.tenant.created` +- `tenancy.tenant.updated` +- `tenancy.tenant.suspended` +- `tenancy.tenant.reactivated` +- `tenancy.tenant.delete_requested` +- `tenancy.tenant.delete_blocked` +- `tenancy.tenant.deleted` + +Policy: + +- `policy.system.updated` +- `policy.tenant.updated` +- `policy.user.updated` +- `policy.group.updated` +- `policy.campaign.updated` +- `policy.effective_policy.changed` + +Files: + +- `files.file.uploaded` +- `files.file.renamed` +- `files.file.deleted` +- `files.file.frozen` +- `files.share.created` +- `files.share.revoked` +- `files.connector.imported` +- `files.connector.access_denied` + +Mail: + +- `mail.profile.created` +- `mail.profile.updated` +- `mail.profile.credentials_rotated` +- `mail.profile.tested` +- `mail.message.sent` +- `mail.message.send_failed` +- `mail.imap.appended` +- `mail.imap.append_failed` +- `mail.mailbox.message_seen` + +Campaign: + +- `campaign.created` +- `campaign.version.created` +- `campaign.validated` +- `campaign.built` +- `campaign.reviewed` +- `campaign.queued` +- `campaign.send_started` +- `campaign.recipient_attempted` +- `campaign.recipient_delivered` +- `campaign.recipient_failed` +- `campaign.paused` +- `campaign.resumed` +- `campaign.cancelled` +- `campaign.report.exported` + +## Event Payload Rules + +- Payloads must be JSON-serializable. +- Use stable IDs, not ORM objects. +- Include tenant ID when tenant-scoped. +- Include actor/principal references only as DTOs or primitive IDs. +- Do not include secrets, raw message bodies, full recipient lists, or file + content. +- Put large evidence in owning module storage and reference it by ID. diff --git a/docs/GITEA_ISSUES.md b/docs/GITEA_ISSUES.md index 382a3e4..a34c882 100644 --- a/docs/GITEA_ISSUES.md +++ b/docs/GITEA_ISSUES.md @@ -33,7 +33,15 @@ For a shared credentials file outside the target repository, pass `--env-file`: ./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply ``` -Create a Gitea token with issue read/write access and label-management permission for the repository. On scoped-token instances, this usually means issue read/write and, if label writes are rejected, repository write permission too. +Create a Gitea token with issue read/write access and label-management +permission for the repository. On scoped-token instances, this usually means +issue read/write and, if label writes are rejected, repository write permission +too. + +For GovOPlaN repositories, prefer organization labels for the shared taxonomy. +Creating or updating organization labels requires a token with +`write:organization`. Repository label management only needs repository label +permission, but it duplicates the taxonomy into each repository. Preview and apply labels: @@ -43,6 +51,19 @@ cd /mnt/DATA/git/govoplan-core ./scripts/gitea-sync-labels.py --apply ``` +Preview and apply the shared taxonomy as organization labels: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env +./scripts/gitea-sync-labels.py --scope organization --env-file /home/zemion/.config/gitea/gitea.env --apply +``` + +The import helpers resolve repository labels and organization labels. Repository +labels win when a repository defines the same name locally, but a repository does +not need a local copy of every shared `type/*`, `status/*`, `priority/*`, +`module/*`, `area/*`, `source/*`, or `codex/*` label. + After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`. ## Multiple Repositories And Workspaces @@ -167,7 +188,10 @@ For a broader project import across all local repositories hosted on `git.add-id ./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply ``` -It scans repository and product-directory files with backlog-like names, creates the shared generic labels where needed, imports missing open work, and deduplicates reruns by hidden fingerprint and normalized title. +It scans repository and product-directory files with backlog-like names, resolves +shared labels from the organization label catalogue where available, creates only +missing fallback repository labels when needed, imports missing open work, and +deduplicates reruns by hidden fingerprint and normalized title. ## Mirroring Project Docs Into Gitea Wikis @@ -184,6 +208,13 @@ Apply the wiki mirror: ./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply ``` +After renaming or deleting repository docs, prune previously managed wiki pages +that no longer have a source file: + +```bash +./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --repo govoplan-core --prune-managed --apply +``` + The default apply path uses the Gitea wiki git repository, not one REST API request per page. It keeps a local checkout cache below `/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and diff --git a/docs/SYSTEM_GOVERNANCE_MANIFEST.md b/docs/GOVERNANCE_MODEL.md similarity index 52% rename from docs/SYSTEM_GOVERNANCE_MANIFEST.md rename to docs/GOVERNANCE_MODEL.md index 4b37741..2fc93dd 100644 --- a/docs/SYSTEM_GOVERNANCE_MANIFEST.md +++ b/docs/GOVERNANCE_MODEL.md @@ -1,11 +1,11 @@ -# Multi Seal Mail - Current System and Tenant Governance Model +# GovOPlaN Governance Model -**Updated:** 2026-06-16 -**Current migration head:** `f5a6b7c8d9e0` +**Updated:** 2026-07-09 ## Governance Rule -System policy is authoritative for tenants and all lower levels. Each lower level may only narrow what it inherits: +System policy is authoritative for tenants and all lower levels. Each lower +level may only narrow what it inherits: ```text system @@ -14,52 +14,65 @@ system -> campaign ``` -Lower levels do not widen privileges, allowed profiles, retention durations or credential rights granted by a higher level. +Lower levels do not widen privileges, allowed profiles, retention durations, or +credential rights granted by a higher level. ## Administration Structure +GovOPlaN separates system administration from scoped configuration: + ```text -SYSTEM -- Settings -- Retention -- Mail servers +ADMINISTRATION +- Modules +- Packages +- Maintenance +- Changes + +GLOBAL - Tenants -- Users -- Groups -- System roles -- Tenant roles -- Audit +- Roles +- Groups and users +- File connectors +- Mail servers +- API keys +- Retention TENANT -- Settings boundary -- Users -- Groups - Roles -- API keys +- Groups and users +- File connectors - Mail servers +- API keys - Retention -- Audit - -USER -- User mail -- User retention GROUP -- Group mail -- Group retention +- File connectors +- Mail servers +- API keys +- Retention + +USER +- File connectors +- Mail servers +- API keys +- Retention ``` -There is no separate System access page. Compatibility access scopes remain in the backend for assignment/read boundaries. +System access scopes remain in the backend for assignment/read boundaries, but +the UI should present the configuration hierarchy rather than a separate +"system access" concept. ## Tenant Governance -System settings define tenant defaults and whether tenants may narrow selected options. Tenant overrides can only restrict: +System settings define tenant defaults and whether tenants may narrow selected +options. Tenant overrides can only restrict: - custom groups; - custom roles; - tenant API keys. -The backend enforces that tenant governance cannot widen system-denied privileges. +The backend enforces that tenant governance cannot widen system-denied +privileges. ## Mail-Profile Governance @@ -73,7 +86,10 @@ group campaign ``` -Effective campaign profile availability follows campaign ownership. A campaign owned by a user resolves through system, tenant, that user and campaign policy. A group-owned campaign resolves through system, tenant, that group and campaign policy. +Effective campaign profile availability follows campaign ownership. A campaign +owned by a user resolves through system, tenant, that user, and campaign +policy. A group-owned campaign resolves through system, tenant, that group, and +campaign policy. Policy semantics: @@ -81,12 +97,18 @@ Policy semantics: - lower levels can further restrict the set; - forced profiles mean the lower level must choose from the forced set; - a forced set with one profile effectively enforces that profile; -- campaign-level profile creation is allowed only if the effective policy permits it; -- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels must inherit profile credentials, may inherit profile credentials, or must provide local credentials; -- the lower-level override switch for `smtp_credentials.inherit` and `imap_credentials.inherit` controls whether descendants may change that inheritance decision; +- campaign-level profile creation is allowed only if the effective policy + permits it; +- SMTP/IMAP credentials use one inheritance decision per protocol: lower levels + must inherit profile credentials, may inherit profile credentials, or must + provide local credentials; +- the lower-level override switch for `smtp_credentials.inherit` and + `imap_credentials.inherit` controls whether descendants may change that + inheritance decision; - deny patterns always win over allow patterns; - empty or `*` allowlist means allow all except denied; -- non-empty allowlist means at least one allow rule must match and no deny rule may match. +- non-empty allowlist means at least one allow rule must match and no deny rule + may match. Pattern targets: @@ -98,7 +120,23 @@ From header recipient domains ``` -Ownership transfer is intentionally deferred as a two-step workflow: original owner initiates, new owner accepts and reselects/repairs the mail profile if their effective policy requires it. +Ownership transfer is intentionally deferred as a two-step workflow: original +owner initiates, new owner accepts and reselects/repairs the mail profile if +their effective policy requires it. + +## File-Connector Governance + +File connector profiles and credentials are separated. Profiles describe +external endpoints; credentials bind authentication material and policy to a +scope. Concrete linked folders appear as file spaces in the files module. + +Governance follows the same inheritance shape as mail: + +- system and tenant policy can permit, require, or forbid lower-level + connections/credentials; +- user or group ownership controls which spaces appear to principals; +- spaces inherit endpoint and credential policy from their connector profile; +- connector health and credential tests must not reveal plaintext secrets. ## Retention Governance @@ -120,20 +158,27 @@ Rules: - system may set concrete defaults or unlimited retention; - system exposes allow-limiting toggles per field; -- tenants, users/groups and campaigns may only shorten inherited retention where the parent allows limiting; +- tenants, users/groups, and campaigns may only shorten inherited retention + where the parent allows limiting; - blank lower-level values inherit; -- mock mailbox retention is currently system-level because mock mailbox records do not yet carry tenant/campaign ownership metadata; -- dry-run/apply retention actions report affected classes before destructive cleanup. +- mock mailbox retention is currently system-level because mock mailbox records + do not yet carry tenant/campaign ownership metadata; +- dry-run/apply retention actions report affected classes before destructive + cleanup. -## Role Definitions and Assignments +## Role Definitions And Assignments -### System roles +### System Roles -System roles define instance-wide permissions. `system:*` is stored as one wildcard and displayed as granting the full system catalogue. System owner is protected. +System roles define instance-wide permissions. `system:*` is stored as one +wildcard and displayed as granting the full system catalogue. System owner is +protected. -### Tenant roles +### Tenant Roles -Tenant roles can be system-governed templates or tenant-local definitions, subject to system tenant-governance settings and actor delegation ceilings. Wildcard counts are expanded against the canonical tenant catalogue. +Tenant roles can be system-governed templates or tenant-local definitions, +subject to system tenant-governance settings and actor delegation ceilings. +Wildcard counts are expanded against the canonical tenant catalogue. ## Audit Access @@ -144,15 +189,17 @@ system audit -> system:audit:read tenant audit -> active tenant + audit:read ``` -Audit pages use server pagination, filtering and bounded grids. +Audit pages use server pagination, filtering, and bounded grids. ## Tenant Switching -Tenant switching preserves the current URL when possible and falls back when a route/resource is not accessible in the new tenant context. +Tenant switching preserves the current URL when possible and falls back when a +route/resource is not accessible in the new tenant context. -The tenant selector is hidden for ordinary single-tenant accounts and visible for multi-tenant or system tenant-management contexts. +The tenant selector is hidden for ordinary single-tenant accounts and visible +for multi-tenant or system tenant-management contexts. -## DataGrid Contract in Administration +## Administration DataGrid Contract Admin lists use bounded container grids: @@ -164,14 +211,12 @@ Admin lists use bounded container grids: - sticky headers where needed; - server pagination for audit. -## Still Deferred +## Deferred Work - real SMTP/IMAP test-bed verification and operator runbook; - recipient import with column mapping; -- Seafile/external connector governance; -- system/tenant/group/user file-space hierarchy and external storage hierarchy; - session/device revocation UI; -- backup/restore, monitoring and update procedures; +- backup/restore, monitoring, and update procedures; - DSAR workflows and evidence bundle verifier; - campaign ownership transfer workflow; - policy impact analysis before delete/disable/unshare/change; diff --git a/docs/GOVOPLAN_MASTER_ROADMAP.md b/docs/GOVOPLAN_MASTER_ROADMAP.md new file mode 100644 index 0000000..7cb72ab --- /dev/null +++ b/docs/GOVOPLAN_MASTER_ROADMAP.md @@ -0,0 +1,663 @@ +# GovOPlaN Master Roadmap + +This roadmap is the durable product north star and sequencing guide for +GovOPlaN as a modular platform for administrative operations. It keeps the +product moving without turning every possible public-sector need into an +immediate implementation track. + +Use this document for product direction, sequencing, and module routing. Issues +are the active backlog; this document is durable planning context and should be +mirrored to the Gitea wiki. + +## Product Thesis + +GovOPlaN should become a configurable operations platform for public +institutions. It should help an institution model real administrative +procedures, connect existing systems, keep durable evidence, and explain the +configured system to users. + +The goal is not to replace every existing system. GovOPlaN should connect +existing systems, provide better workflows where the current landscape is weak, +and make administrative processes configurable, auditable, and reusable. + +The product should first provide a reliable administrative spine: + +- identities, roles, tenants, policy, audit, and governance +- forms, files, cases, workflow, tasks, templates, and records +- postboxes, notifications, mail, portal, appointments, and booking +- identity trust, role-bound postboxes, and eventually encrypted administrative + communication +- configuration packages that assemble modules into repeatable procedures +- docs that explain the configured system, not the full theoretical product + +Domain modules should come after the spine can run a reference procedure end to +end. + +The platform should be a governance-capable runtime for modules, connectors, +configuration, and administrative decisions. The kernel must stay free of domain +semantics while still providing the contracts needed for modules to explain what +they do, what they require, which effects they create, and how operators can +verify or reverse those effects. + +## Design Principles + +- Modules must stay independently installable, enableable, and disableable. +- Cross-module behavior should use core-mediated capabilities, commands, + events, DTOs, and UI contribution points rather than direct imports. +- Configuration packages should turn installed modules into concrete, reusable + administrative processes. +- Operators should be able to configure the platform through the UI. +- Every powerful configuration path needs preflight, preview, audit, rollback, + RBAC, and policy checks. +- Context, decision, consequence, and traceability must be visible together for + consequential actions. The full doctrine lives in + `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md`. +- Automation must use governed action/effect contracts, not hidden side + effects. The first automation layer is defined in + `ACTION_EFFECT_AUTOMATION_LAYER.md` and should start in `govoplan-workflow` + unless a separate automation module becomes justified. +- Encrypted postboxes are a strategic target. Early postbox, access, and + identity-trust contracts should stay compatible with the E2EE architecture in + `POSTBOX_E2EE_ARCHITECTURE.md`. +- Integration should be a first-class product path: connect to existing + systems, consume their data, and publish governed outputs back to them. +- GovOPlaN should scale from a small local installation to a larger deployment + with separately scalable web, API, worker, storage, and database components. + +## User Experience Direction + +GovOPlaN should expose the full power of the platform without forcing +non-technical users to face every field, flag, and internal representation at +once. The default experience should feel guided, explainable, and calm. Expert +depth should remain available, but it should be layered behind deliberate +interaction patterns. + +Core UX rules: + +- Use progressive disclosure. Common decisions stay visible; advanced, + hazardous, or rarely used options live in collapsed panels, secondary steps, + or explicit advanced areas. +- Do not use raw JSON as the primary configuration UI. Every configurable value + should have an appropriate control, validation, and plain-language help. + Import/export and diagnostics may show JSON as a secondary artifact. +- Prefer guided flows over option dumps. Connector setup, package import, + module installation, policy changes, and destructive operations should use + wizards that explain what is happening, why it matters, and what will happen + next. +- Discover values when the system can infer them. For example, a Nextcloud file + connection should start with the base URL, discover the WebDAV endpoint, and + fill technical fields for review instead of asking the user to know them + upfront. +- Make explanations always available without making every screen verbose. + Inline helper text should be short; richer explanations should be reachable + through expandable help, tooltips, side panels, or review steps. +- Explain blocked actions in actionable language. A disabled control or failed + step should say what is missing, who can fix it, and where to go, for example + "A system administrator must allow this provider" or "Configure the provider + in Settings > File Providers before linking a folder here." +- Reuse visual language and placements consistently. Similar configuration, + policy, connection, credential, review, and confirmation flows should share + components, button placement, modal behavior, problem lists, and empty/error + states. +- Use modals and step flows for focused creation/editing where they reduce page + clutter. Reserve large always-open pages for overview, comparison, and + repeated administration work. +- Treat diagnostics as product UX. Validation results, preflight blockers, + policy explanations, permission denials, and missing capabilities should be + understandable to a non-technical operator before exposing internal details. + +This is a product quality gate. New admin/configuration surfaces should not be +considered complete if they expose all options at once, require JSON editing, +hide why an action is unavailable, or use a one-off layout where a shared +pattern exists. + +## Focus Rules + +1. Build one reference journey per wave. +2. Do not implement a module because the repository exists. +3. Do not add module-to-module imports for optional behavior. +4. Every new domain module must justify its own semantics beyond `cases`, + `workflow`, `tasks`, `forms`, and `files`. +5. Prefer connector first when an external specialist system is likely to remain + the system of record. +6. Keep active work in Gitea issues; keep durable context in docs and synced + wiki pages. +7. A module moves from scaffold to implementation only when it has an owner, + reference journey, boundary notes, capability contracts, and testable MVP. + +## Capability Map + +| Capability | Likely owner | +| --- | --- | +| Public application entry point | `govoplan-portal` | +| Structured forms and validation | `govoplan-forms` | +| Uploaded files and managed storage | `govoplan-files` | +| Case record and lifecycle | `govoplan-cases` | +| Workflow transitions and automation | `govoplan-workflow` | +| Action/effect catalogue and automation runner | first `govoplan-workflow`; possible future `govoplan-automation` if it outgrows workflow | +| Internal work queues and tasks | `govoplan-tasks` | +| Appointment proposals and booking | `govoplan-appointments`, `govoplan-calendar` | +| Postbox, email, and notifications | `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications` | +| Identity trust, device keys, and encrypted postbox key contracts | `govoplan-identity-trust`, `govoplan-access`, `govoplan-postbox` | +| Service directory/catalog | `govoplan-portal` | +| Permit/document generation | `govoplan-templates`, `govoplan-dms` | +| Payment capture and accounting handoff | `govoplan-payments`, `govoplan-ledger` | +| Roles, permissions, tenants, policy, audit | `govoplan-access`, `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit` | +| External software integration | `govoplan-connectors` | +| Recurring extraction and transformation | possible future `govoplan-datasources`, possible future `govoplan-dataflow` | +| Reports, BI, and management visibility | `govoplan-reporting` | + +## Configuration And Safety Target + +The long-term target is that operators configure the platform through the UI +instead of editing files for normal operation. + +UI-managed configuration should include: + +- module installation, enablement, lifecycle state, and health +- tenants, users, groups, roles, policies, and permissions +- connectors, credentials, secret references, and external service tests +- workflows, forms, templates, task queues, schedules, and notifications +- configuration package import/export and environment-specific data collection +- retention, audit, privacy, maintenance mode, and safety controls +- deployment-visible settings such as public URLs, mail senders, storage + profiles, queues, and worker capabilities + +Safety controls should include dry-run plans, field-level validation, policy +explanations, two-person approval for destructive changes, versioned +configuration history, rollback paths, audit events, and maintenance-mode +guards. + +The initial safety metadata contract lives in +`govoplan_core.core.configuration_safety`. It classifies known configuration +fields as UI-managed or deployment-managed, assigns risk levels, marks secret +handling as reference-only or env-only, and declares dry-run, policy +explanation, audit, approval, rollback-history, maintenance-mode, and RBAC +requirements. Admin UI editors should consume this metadata before exposing +powerful settings. + +The initial executable guardrail path is `plan_configuration_change(...)`, +exposed through: + +- `GET /api/v1/admin/configuration-safety` +- `POST /api/v1/admin/configuration-safety/plan` + +The planner reports missing scopes, dry-run requirements, maintenance-mode +requirements, two-person approval status, secret-reference violations, +rollback-history requirements, policy explanations, and audit event names before +an editor applies a high-impact configuration change. + +## Reference Journeys + +The roadmap should be driven by three journeys. + +### Journey 1: Permit To Payment + +This is the primary public-administration journey. + +1. A person applies for a permit through the public portal. +2. The applicant uploads required files and submits structured form data. +3. Submission creates a case, a workflow instance, and an internal task. +4. Completing the task creates a postbox message, a notification, and an email + notification with an appointment proposal. +5. The applicant accepts an appointment, which updates the calendar and the + workflow state. +6. During the appointment, the case is opened and the permit is generated from + a governed template. +7. The payment is processed and linked to the case and accounting handoff. +8. The permit, payment evidence, communication history, audit trail, retention + state, and records evidence remain available according to policy. + +This journey proves the platform can coordinate modules without core knowing +module internals. + +### Journey 2: Training To Certificate + +This is the best university-administration and internal-administration journey. + +1. Course or training offer is planned. +2. Room, trainer, resource, and capacity are booked. +3. Participants register or are assigned. +4. Attendance is tracked. +5. Certificate or participation confirmation is issued. +6. Evidence remains available through records, files, audit, and docs. + +This journey keeps `booking`, `resources`, `learning`, and `certificates` +focused instead of becoming broad ERP replacements. + +### Journey 3: Report To Resolution + +This is the internal operations and municipal issue-reporting journey. + +1. A person reports an issue. +2. The issue is triaged into helpdesk, facilities, assets, or a case. +3. Work is assigned, tracked, and escalated. +4. Evidence, communication, and status updates are preserved. +5. Reports show workload, SLA, recurring problems, and completion. + +This journey prevents `helpdesk`, `issue-reporting`, `facilities`, and `assets` +from becoming disconnected ticket silos. + +## Roadmap Waves + +### Wave 0: Platform Spine + +Goal: make the platform safe to configure and extend. + +Refine: + +- `govoplan-core`: module discovery, capabilities, events, migrations, release + catalog, configuration package runtime, WebUI shell. +- `govoplan-access`: identities, sessions, API keys, users, groups, roles, + memberships, function assignments, delegation, RBAC decisions. +- `govoplan-tenancy`: tenant and organizational-unit boundaries. +- `govoplan-identity-trust`: initial trust contracts for device keys, public key + directory, assurance, and later encrypted postbox key access. +- `govoplan-policy`: policy sources, policy decisions, retention inputs. +- `govoplan-audit`: audit sink, audit routes, trace context, retention + cooperation. +- `govoplan-admin`: UI-managed configuration with preflight, rollback, audit, + and approval controls. +- `govoplan-dashboard`: configurable user home assembled from module-provided + widgets, with core providing only a minimal fallback when the module is absent. +- `govoplan-docs`: configured-system documentation and evidence-aware help. +- `govoplan-ops`: deployment profiles, health checks, worker split, sizing + assumptions. + +Do not expand domain scope in this wave. The output is a dependable platform +surface for later modules. + +Exit criteria: + +- module enablement and capability lookup are stable +- configuration package preflight works for at least one simple package +- audit and policy decisions are visible in admin flows +- access distinguishes identity, account, function, role, and right in durable + contracts +- action/effect automation contracts are specified before hidden side effects + spread across modules +- docs can show installed/enabled modules and configured routes + +### Wave 1: Permit-To-Payment MVP + +Goal: one complete public-administration process. + +Create or refine in this order: + +1. `govoplan-forms-runtime`: form submissions, drafts, validation, attachments, + and submission evidence. +2. `govoplan-portal`: public authenticated and unauthenticated entry points. +3. `govoplan-files`: upload, evidence links, file spaces, and permissioned + access. +4. `govoplan-cases`: case record, status, assignments, deadlines, and case + evidence. +5. `govoplan-workflow`: state machine, transitions, commands, and module + handoff. +6. `govoplan-tasks`: work queues, assignments, due dates, and follow-ups. +7. `govoplan-templates`: permit/decision document generation. +8. `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications`: applicant and + internal communication, with the postbox model compatible with later E2EE + and role-bound access. +9. `govoplan-calendar`, `govoplan-appointments`, `govoplan-booking`: appointment + or booking handoff for the reference process. +10. `govoplan-payments`, `govoplan-ledger`, `govoplan-xrechnung`: payment + capture, payment evidence, and accounting/e-invoice handoff. + +Exit criteria: + +- a permit-to-payment package can be installed in a local demo +- every step has audit evidence +- every module integration uses capabilities, events, or DTOs +- user-facing documentation describes only the configured process + +### Wave 2: Booking And Resource Operations + +Goal: make time, capacity, and resource allocation a reusable product area. + +Create or refine in this order: + +1. `govoplan-booking`: booking rules, capacity, quotas, waitlists, cancellation, + attendance, no-shows, and approvals. +2. `govoplan-resources`: rooms, equipment, vehicles, counters, labs, capacity, + availability, and maintenance blocks. +3. `govoplan-calendar`: event and availability integration. +4. `govoplan-appointments`: appointment-specific booking surfaces. +5. `govoplan-facilities`: buildings, rooms, maintenance, access zones, + inspections, and defects. +6. `govoplan-assets`: inventory, assignment, lifecycle, maintenance, and handover + evidence. + +Reference journey: reserve a room/resource for a training or appointment and +preserve all booking evidence. + +Exit criteria: + +- bookable resources are not hardcoded into appointments +- booking decisions are explainable and auditable +- resources can be blocked by maintenance or facility status + +### Wave 3: Training And Certificates + +Goal: support university-style and internal public-sector training without +building a full learning-management system first. + +Create or refine in this order: + +1. `govoplan-learning`: course planning, course booking, participant lists, + trainers, attendance, evaluations, and learning records. +2. `govoplan-certificates`: participation confirmations, certificates, + verification, revocation, and evidence links. +3. `govoplan-booking`, `govoplan-resources`, `govoplan-calendar`: reuse booking + and room/resource allocation. +4. `govoplan-templates`, `govoplan-files`, `govoplan-records`: certificate + generation and durable retention. + +Reference journey: plan a course, book resources, register participants, track +attendance, and issue a certificate. + +Exit criteria: + +- attendance can produce certificate eligibility +- certificates can be verified later +- course operations do not depend on university-specific assumptions + +### Wave 4: Report-To-Resolution Operations + +Goal: cover internal support and public issue reporting. + +Create or refine in this order: + +1. `govoplan-issue-reporting`: public/internal reports, categories, intake, + location, evidence, and triage. +2. `govoplan-helpdesk`: service desk tickets, queues, SLAs, assignments, + escalation, and resolution evidence. +3. `govoplan-facilities` and `govoplan-assets`: issue handoff to maintenance or + asset lifecycle. +4. `govoplan-cases`, `govoplan-workflow`, `govoplan-tasks`: escalation into + formal administrative matters. +5. `govoplan-reporting`: workload, SLA, recurring defects, and status reports. + +Reference journey: report a facility issue, triage it, assign work, resolve it, +and report recurring defects. + +Exit criteria: + +- issue reporting is not just a generic form inbox +- helpdesk tickets can remain lightweight unless formal case handling is needed +- operational metrics exist without custom SQL + +### Wave 5: Records, DMS, Transparency + +Goal: make evidence legally and organizationally durable. + +Create or refine in this order: + +1. `govoplan-dms`: document lifecycle, collaboration, versions, locks, approvals, + and DMS connectors. +2. `govoplan-records`: file plans, classification, retention schedules, disposal + holds, archive handoff, and records evidence. +3. `govoplan-policy` and `govoplan-audit`: retention policy, legal hold, audit + traceability. +4. `govoplan-search`: permissioned cross-module discovery. +5. `govoplan-transparency`: FOI/public-records requests, redaction, publication, + and disclosure evidence. + +Reference journey: close a case, classify records, apply retention, and answer a +transparency request. + +Exit criteria: + +- files, documents, and records are not treated as the same concept +- transparency disclosure can redact and explain evidence +- retention and archive handoff are policy-governed + +### Wave 6: Finance, Procurement, Contracts, Grants + +Goal: cover the back-office procedures around spending, obligations, funding, +and revenue without replacing a finance system too early. + +Create or refine in this order: + +1. `govoplan-procurement`: purchase requests, approvals, vendor comparison, + tender references, goods receipt, and handoff. +2. `govoplan-contracts`: contract register, obligations, renewals, reminders, + and responsible units. +3. `govoplan-grants`: funding programs, applications, eligibility, awards, + milestones, claims, and reporting duties. +4. `govoplan-payments`, `govoplan-ledger`, `govoplan-xrechnung`: payment, + accounting, and e-invoice handoff. +5. `govoplan-erp`: integration with the actual system of record, not a full ERP + replacement unless later justified. + +Reference journey: purchase or grant approval through obligation tracking and +financial handoff. + +Exit criteria: + +- ERP remains an integration point unless GovOPlaN must own semantics +- contracts and grants produce obligations, deadlines, and reporting tasks +- financial evidence links back to cases, files, and audit + +### Wave 7: Institutional Governance And Participation + +Goal: support public bodies, municipalities, ministries, and universities in +formal coordination. + +Create or refine in this order: + +1. `govoplan-committee`: committees, boards, councils, agendas, minutes, + decisions, voting, and follow-up tasks. +2. `govoplan-consultation`: hearings, public consultations, stakeholder + feedback, formal comments, response matrices, and publication workflows. +3. `govoplan-campaign`: communication campaigns, recipients, templates, review, + sending control, and reports. +4. `govoplan-addresses`: persons, organizations, contacts, distribution lists, + and recipient import/export. +5. `govoplan-portal` and `govoplan-transparency`: public participation and + publication surfaces. + +Reference journey: prepare a committee decision, publish consultation material, +collect feedback, document the decision, and assign follow-up tasks. + +Exit criteria: + +- decisions create durable tasks and records +- consultations can publish evidence without exposing restricted material +- campaign and address behavior stays optional and capability-based + +### Wave 8: Integration, Data, And Reporting + +Goal: connect the platform to real institutional landscapes. + +Refine: + +- `govoplan-connectors`: integration catalog, connector metadata, adapter + lifecycle, test harnesses. +- `govoplan-idm`, `govoplan-identity-trust`: identity provider, directory, and + assurance integration. +- `govoplan-fit-connect`, `govoplan-xoev`, `govoplan-xta-osci`: public-sector + transport and standards integration. +- `govoplan-reporting`: operational reporting, scheduled outputs, exports, and + dashboard data. +- `govoplan-search`: permissioned cross-module discovery. + +Create only when justified: + +- `govoplan-datasources`: source catalog, connection profiles, schema discovery, + freshness, provenance. +- `govoplan-dataflow`: transformations, validation, lineage, scheduled runs, + publication outputs. +- `govoplan-projects`: only if OpenProject connectors and existing cases/tasks + cannot cover the required semantics. + +Reference journey: monthly data extraction, transformation, validation, approval, +publication, and reporting. + +Recurring extraction/transformation should start as a configuration package +across connectors, files, workflow, reporting, and templates. The package should +register sources, declare schemas, define mapping/validation versions, schedule +runs, produce previewable diffs, write governed outputs, and preserve lineage, +hashes, operator actions, and audit evidence. Create `govoplan-datasources` or +`govoplan-dataflow` only after this work exposes repeated contracts that do not +belong to existing modules. + +Exit criteria: + +- connector catalog exists before building many adapters +- dataflow is created only after recurring transformation becomes product + behavior +- reporting consumes governed sources with provenance + +## Implementation Gates + +Before a module receives implementation work, it needs: + +- one reference journey step it owns +- ownership and boundary notes +- initial manifest and permission list +- capability contracts or API DTOs +- audit and policy expectations +- Gitea issues for MVP tasks +- docs page that can sync to the wiki +- focused tests that can run from the core environment + +Before a module becomes release-included, it needs: + +- installable Python package metadata where applicable +- WebUI package metadata where applicable +- module manifest entry point +- release catalog entry +- smoke test or permutation test +- no required imports from optional modules + +## Priority Order Summary + +1. Stabilize the platform spine. +2. Deliver permit-to-payment MVP. +3. Build booking and resource operations. +4. Add learning and certificates. +5. Add issue reporting and helpdesk. +6. Add records, DMS, search, and transparency. +7. Add procurement, contracts, grants, and finance handoff. +8. Add committee and consultation workflows. +9. Expand integration, dataflow, reporting, and operations. + +## Deliberate Deferrals + +Defer these until a reference journey proves the need: + +- full ERP replacement +- native project management beyond connector support +- broad BI/dataflow platform +- every possible public-sector protocol adapter +- rich LMS behavior beyond training administration +- full qualified digital signing/trust services beyond the identity-trust and + encrypted-postbox contracts needed for early architecture safety +- mobile apps +- AI assistants embedded into workflows + +These may become important, but they should not distract from the first complete +administrative journeys. + +## Module And Integration Routing + +This table maps current module and integration ideas to existing GovOPlaN +repositories or to explicit missing-module decisions. + +| Idea | Owner | Tracking | +| --- | --- | --- | +| Government operations backbone reference model | `govoplan-core` | `add-ideas/govoplan-core#213` | +| Permit-to-payment configuration package | `govoplan-core` plus participating modules | `add-ideas/govoplan-core#214` | +| Fully UI-managed configuration with safety controls | `govoplan-admin`, `govoplan-core`, `govoplan-policy`, `govoplan-access`, `govoplan-audit` | `add-ideas/govoplan-core#218` | +| Access as a module | `govoplan-access` | `add-ideas/govoplan-access#7` | +| Interface ethics and decision-consequence doctrine | `govoplan-core` plus all UI-owning modules | `add-ideas/govoplan-core#227` | +| Action/effect automation layer | first `govoplan-workflow`; possible future `govoplan-automation` | `add-ideas/govoplan-workflow#1` | +| E2EE role/function postbox architecture | `govoplan-postbox`, `govoplan-identity-trust`, `govoplan-access`, `govoplan-policy`, `govoplan-audit` | `add-ideas/govoplan-postbox#15`, `add-ideas/govoplan-identity-trust#1` | +| Identity, account, function, role, right semantic model | `govoplan-access` | `add-ideas/govoplan-access#9` | +| Role-based service directory/catalog | `govoplan-portal` | `add-ideas/govoplan-portal#1` | +| Unified inbox across tasks, postbox, notifications, and portal | `govoplan-core` coordination plus owning modules | `add-ideas/govoplan-tasks#1`, `add-ideas/govoplan-notifications#1` | +| OpenProject API / project management connector | `govoplan-connectors` | `add-ideas/govoplan-connectors#1` | +| Native project-management module decision | connector-first through `govoplan-connectors`; no native project module yet | `add-ideas/govoplan-core#196`, `add-ideas/govoplan-connectors#1` | +| Datasources for databases, CSV, files, APIs | no repository yet; start with connectors/files/reporting and create `govoplan-datasources` only after the first package proves shared source-catalog ownership | `add-ideas/govoplan-core#197` | +| Dataflow for pipelines, BI, publication | no repository yet; start with workflow/reporting/connectors and create `govoplan-dataflow` only after repeated pipeline/lineage contracts emerge | `add-ideas/govoplan-core#198` | +| Monthly datasource and transformation workflows | first as configuration package across connectors, files, workflow, reporting, and templates | `add-ideas/govoplan-core#216` | +| Templates for letters, emails, forms, reports | `govoplan-templates`, separate from reporting | `add-ideas/govoplan-core#190`, `add-ideas/govoplan-templates#1` | +| Reporting and BI | `govoplan-reporting`, separate from templates | `add-ideas/govoplan-core#190`, `add-ideas/govoplan-reporting#1` | +| File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `add-ideas/govoplan-files#15` | +| Public-sector software integration catalogue | `govoplan-connectors` with core strategy index | `add-ideas/govoplan-core#191`, `add-ideas/govoplan-connectors#2` | +| Public-sector integration landscape catalogue | `govoplan-connectors` with core tracking | `add-ideas/govoplan-core#215` | +| Cases module concept | `govoplan-cases` | `add-ideas/govoplan-core#174` | +| Workflow module concept | `govoplan-workflow` | `add-ideas/govoplan-core#175` | +| Connectors module concept | `govoplan-connectors` | `add-ideas/govoplan-core#176` | +| Adrema-style address and distribution-list management | `govoplan-addresses` | `add-ideas/govoplan-addresses#1` | +| Consume sources and become a governed source | `govoplan-connectors` plus possible future `govoplan-dataflow` | `add-ideas/govoplan-connectors#3`, `add-ideas/govoplan-core#198` | +| Governed connector configuration, dry-run, and simulation runtime | `govoplan-connectors` | `add-ideas/govoplan-connectors#6` | +| Terminfindung and meeting scheduling polls | `govoplan-scheduling`; calendar primitives remain in calendar | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-scheduling#1` | +| Terminplaner and calendar primitives | `govoplan-calendar` | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-calendar#1` | +| Terminbuchung appointment booking | `govoplan-appointments` | `add-ideas/govoplan-core#193`, `add-ideas/govoplan-appointments#1` | +| Collaborative documents | `govoplan-dms` | `add-ideas/govoplan-dms#1` | +| Forms | `govoplan-forms` for definitions and `govoplan-forms-runtime` for submissions/runtime behavior | `add-ideas/govoplan-core#194`, `add-ideas/govoplan-forms#1` | +| RSS consume and emit | `govoplan-connectors` | `add-ideas/govoplan-connectors#4` | +| LDAP, Active Directory, OpenDesk identity | `govoplan-idm` | `add-ideas/govoplan-idm#1` | +| OpenDesk stack integration map | integration profile across IDM/access, mail/calendar, files/DMS, and connectors; not a monolithic module | `add-ideas/govoplan-core#195`, `add-ideas/govoplan-connectors#5` | +| Open-Xchange mail/groupware | `govoplan-mail` | `add-ideas/govoplan-mail#5` | +| Open-Xchange calendar | `govoplan-calendar` | `add-ideas/govoplan-calendar#2` | +| Scalability profiles and autoscaling readiness | `govoplan-ops`, `govoplan-core` | `add-ideas/govoplan-core#217` | +| Hardware sizing matrix and requirements calculator | `govoplan-ops`, `govoplan-core` | `add-ideas/govoplan-core#219` | +| Collaboration suite integration strategy | `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow`, `govoplan-tasks`, `govoplan-appointments`, `govoplan-calendar` | `add-ideas/govoplan-core#220` | +| Install/runtime configuration contract | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#19` | +| Installer/deployment operator flow | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#26` | +| Production-like deployment documentation | `govoplan-core`, later `govoplan-ops` | `add-ideas/govoplan-core#28` | + +Boundary rationale lives in `MODULE_ARCHITECTURE.md`. Current decisions: + +- templates and reporting are separate modules +- RSS/source consume-publish starts in connectors; datasources/dataflow are not + repositories yet +- calendar, scheduling, and appointments are three separate modules +- forms definitions and forms runtime are separate responsibilities +- OpenDesk is an integration profile across modules, not a monolithic module +- OpenProject is connector-first; no native projects module yet +- public-sector integration strategy stays in core; executable catalogue work + lives in connectors +- encrypted postbox and identity-trust are strategic contracts, not mail-module + behavior +- automation starts as workflow-owned action/effect execution and may split into + a dedicated module only after the runner becomes broader than workflow + +The following modules are intentionally not created yet: + +- `govoplan-datasources` +- `govoplan-dataflow` +- `govoplan-projects` + +Create a repository only after a concrete implementation package proves that +existing connector, files, reporting, workflow, or task ownership is too narrow. + +Core keeps the strategy index in +`PUBLIC_SECTOR_INTEGRATION_STRATEGY.md`: integration postures, default +ownership, and prioritization rules. `govoplan-connectors` owns the detailed +target inventory and connector entry shape in +`govoplan-connectors/docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md`. + +Release composition and tag-only repository handling are documented in +`RELEASE_DEPENDENCIES.md`. + +## Next Practical Work + +The next planning step should create or update Gitea issues for Wave 0 and Wave +1 only. Later waves should stay as roadmap context until the permit-to-payment +MVP is demonstrable. + +Recommended immediate issue buckets: + +- platform spine hardening +- configuration package preflight and rollback +- forms-runtime MVP +- portal submission MVP +- cases/workflow/tasks integration MVP +- template-generated decision document +- postbox/notification handoff +- appointment/booking handoff +- payment evidence handoff +- configured documentation for the reference process diff --git a/docs/GOVOPLAN_MODULE_ROADMAP.md b/docs/GOVOPLAN_MODULE_ROADMAP.md deleted file mode 100644 index 26eb760..0000000 --- a/docs/GOVOPLAN_MODULE_ROADMAP.md +++ /dev/null @@ -1,79 +0,0 @@ -# GovOPlaN Module And Integration Roadmap - -This page maps current module and integration ideas to existing GovOPlaN repositories or to missing-module decisions. Issues are the active backlog. This document is durable routing context and should be mirrored to the Gitea wiki. - -## Current Routing - -| Idea | Owner | Tracking | -| --- | --- | --- | -| Access as a module | `govoplan-access` | `add-ideas/govoplan-access#7` | -| OpenProject API / project management connector | `govoplan-connectors` | `add-ideas/govoplan-connectors#1` | -| Native project-management module decision | `govoplan-core` | `add-ideas/govoplan-core#196` | -| Datasources for databases, CSV, files, APIs | proposed `govoplan-datasources` | `add-ideas/govoplan-core#197` | -| Dataflow for pipelines, BI, publication | proposed `govoplan-dataflow` | `add-ideas/govoplan-core#198` | -| Templates for letters, emails, forms, reports | `govoplan-templates` | `add-ideas/govoplan-templates#1` | -| Reporting and BI | `govoplan-reporting` | `add-ideas/govoplan-reporting#1` | -| File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `add-ideas/govoplan-files#15` | -| Public-sector software integration catalogue | `govoplan-connectors` | `add-ideas/govoplan-connectors#2` | -| Adrema-style address and distribution-list management | `govoplan-addresses` | `add-ideas/govoplan-addresses#1` | -| Consume sources and become a governed source | `govoplan-connectors` plus proposed `govoplan-dataflow` | `add-ideas/govoplan-connectors#3`, `add-ideas/govoplan-core#198` | -| Terminfindung and meeting scheduling polls | `govoplan-scheduling` | local scaffold; remote creation tracked by `add-ideas/govoplan-core#199` | -| Terminplaner and calendar primitives | `govoplan-calendar` | `add-ideas/govoplan-calendar#1` | -| Terminbuchung appointment booking | `govoplan-appointments` | `add-ideas/govoplan-appointments#1` | -| Collaborative documents | `govoplan-dms` | `add-ideas/govoplan-dms#1` | -| Forms | `govoplan-forms` | `add-ideas/govoplan-forms#1` | -| RSS consume and emit | `govoplan-connectors` | `add-ideas/govoplan-connectors#4` | -| LDAP, Active Directory, OpenDesk identity | `govoplan-idm` | `add-ideas/govoplan-idm#1` | -| OpenDesk stack integration map | `govoplan-connectors` | `add-ideas/govoplan-connectors#5` | -| Open-Xchange mail/groupware | `govoplan-mail` | `add-ideas/govoplan-mail#5` | -| Open-Xchange calendar | `govoplan-calendar` | `add-ideas/govoplan-calendar#2` | - -## Proposed Missing Modules - -`govoplan-datasources` should exist only if source catalog ownership becomes broad enough to justify a module separate from connectors and reporting. Candidate responsibilities: - -- source catalogue for SQL databases, CSV/Excel files, uploaded files, APIs, RSS feeds, and governed file locations -- credentials and connection profiles -- schema discovery and refresh cadence -- provenance, freshness, permission boundaries, and audit events - -`govoplan-dataflow` should exist only if pipelines and publication become first-class product behavior. Candidate responsibilities: - -- ingestion, transformation, validation, scheduling, and lineage -- connecting datasources to reports, APIs, RSS, exports, and downstream systems -- the "consume sources, become source" lifecycle -- publication-state and audit integration - -`govoplan-projects` is undecided. The default path should be an OpenProject connector first. A native module is justified only if GovOPlaN must own project semantics beyond cases, tasks, workflow, appointments, documents, and reporting. - -## Boundary Notes - -- `govoplan-connectors` owns protocol and external-system integration strategy. It should not own business semantics once a domain module exists. -- `govoplan-files` owns file storage semantics and file-provider contracts. Remote provider implementations must stay optional. -- `govoplan-dms` owns document lifecycle, collaboration, versions, approvals, locks, retention, and legal hold. -- `govoplan-addresses` owns persons, organizations, postal/email addresses, distribution lists, and recipient import/export. -- `govoplan-calendar` owns events, availability, resources, recurrence, and groupware calendar adapters. -- `govoplan-scheduling` owns meeting scheduling polls, participant availability collection, candidate-slot ranking, and decision handoff. -- `govoplan-appointments` owns public/internal appointment-booking workflows. -- `govoplan-idm` owns directory, provisioning, and identity-provider integration; `govoplan-access` consumes resolved principals and permissions. -- `govoplan-reporting` owns report definitions, BI views, scheduled outputs, and export targets. -- `govoplan-templates` owns reusable renderable templates, not data selection or persistence. - -## Release Tooling Note - -`scripts/push-release-tag.sh` covers the released full-product package set: - -- `govoplan-access` -- `govoplan-admin` -- `govoplan-tenancy` -- `govoplan-policy` -- `govoplan-audit` -- `govoplan-files` -- `govoplan-mail` -- `govoplan-campaign` -- `govoplan-calendar` -- `govoplan-core` - -It also includes existing roadmap/scaffold module repositories such as addresses, appointments, connectors, DMS, forms, IDM, reporting, scheduling, templates, workflow, XOE/V, and XRechnung as tag-only repositories. Tag-only repositories are committed, tagged, and pushed with the same release tag, but they are not added to `requirements-release.txt` or `webui/package.release.json` until they contain installable package metadata. - -Proposed modules without repositories, such as `govoplan-datasources`, `govoplan-dataflow`, and possibly `govoplan-projects`, cannot be included until their repositories are created. diff --git a/docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md b/docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md new file mode 100644 index 0000000..10eef0e --- /dev/null +++ b/docs/INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md @@ -0,0 +1,102 @@ +# GovOPlaN Interface Ethics And Design Doctrine + +This document captures the product-level design doctrine for GovOPlaN. It is +more durable than an individual screen design and should guide admin, +configuration, workflow, policy, portal, and operational UI decisions. + +GovOPlaN is meant to support administrative responsibility. The interface must +therefore make context, decision, consequence, and traceability visible enough +that users can act deliberately instead of being pushed through opaque +automation. + +## Core Doctrine + +1. Context, decision, and consequence belong together. +2. Decisions should not silently happen. +3. Transparency comes before convenience when rights, duties, records, money, + access, or legal effects are involved. +4. Explicit state is preferable to implicit state. +5. Navigation is not consent. +6. Responsibility cannot be delegated to the system. +7. Traceability is part of the action, not a later reporting feature. +8. Context loss is a product defect. + +These rules do not mean every screen should become verbose. They mean the +interface must expose the right explanation at the moment of decision and keep +technical detail available without making it the default surface. + +## Decision Surface Contract + +Any action that changes records, rights, policies, retention, communication, +payments, external systems, or workflow state should answer these questions +before execution: + +- What object, person, organization, or process is affected? +- Which authority or role allows the actor to do this? +- What will change immediately? +- What downstream effects may happen? +- Can the action be undone, superseded, or only corrected later? +- What evidence or audit entry will be created? +- Which policy, configuration, or missing capability blocks the action? +- Who can resolve a blocker? + +The answer may be shown through inline labels, a review step, a side panel, a +problem list, or a confirmation dialog. The important point is that consequence +and responsibility are not hidden behind a generic submit button. + +## Contestability + +Administrative decisions are often contestable or reviewable. GovOPlaN should +therefore preserve the path from input to decision: + +- source data and attachments +- workflow state and task assignment +- policy decisions and source path +- actor and delegation context +- generated document/template version +- external handoff result +- notification or postbox delivery evidence +- retention and record classification state + +Where a user sees a decision, they should be able to reach the provenance that +explains how the system got there. This is especially important for denials, +locks, calculated defaults, generated documents, payment state, retention +state, and access decisions. + +## Anti-Patterns + +Avoid these patterns in GovOPlaN interfaces: + +- Magical buttons that execute multiple side effects without preview. +- Process tunnels that hide where the user is in an administrative procedure. +- Silent automation that changes external systems without an audit-visible + command record. +- Friendly hiding that removes complexity at the cost of obscuring authority, + consequence, or accountability. +- Disabled controls without actionable explanation. +- Configuration screens that ask users to edit raw JSON as the normal path. + +## Automation Rule + +Automation must use the same governed action surface as a human actor. The +system may execute actions as a system actor, but it must still run through +policy checks, capability contracts, audit, idempotency, and failure handling. + +When an automated decision is not clear, GovOPlaN should create a manual +exception, task, or review item instead of guessing silently. + +## Relationship To UI Components + +Shared components should make this doctrine easy to follow: + +- preflight and problem-list components for blockers +- policy source path and effective decision displays +- action review panels for consequence preview +- audit/provenance links on decision outputs +- guided dialogs for risky configuration +- disabled-action explanations with actor and next step +- confirmation dialogs that distinguish reversible, corrective, and destructive + actions + +The UI/UX decision ledger defines concrete implementation rules. This doctrine +defines why those rules exist. diff --git a/docs/MODULE_ARCHITECTURE.md b/docs/MODULE_ARCHITECTURE.md index 1598fa3..6d452b8 100644 --- a/docs/MODULE_ARCHITECTURE.md +++ b/docs/MODULE_ARCHITECTURE.md @@ -2,10 +2,17 @@ GovOPlaN is structured as a platform kernel plus installable modules. The kernel starts and composes the platform. Modules own product behavior and contribute backend routes, database metadata, permissions, WebUI routes, navigation metadata, capabilities, and events. -The current package name is still `govoplan-core`, but the architecture target is a smaller kernel. Access, tenancy, policy, audit, and admin semantics are platform-module responsibilities and should be extracted in stages. +The current package name is still `govoplan-core`, but the architecture target is a smaller kernel. Access, tenancy, policy, audit, and admin semantics are platform-module responsibilities. -The concrete access/auth/RBAC extraction path is tracked in -[`ACCESS_EXTRACTION_PLAN.md`](ACCESS_EXTRACTION_PLAN.md). +Access extraction is complete enough that current ownership is described here, +in [`ACCESS_RBAC_MODEL.md`](ACCESS_RBAC_MODEL.md), and in the +`govoplan-access` repository docs. +The event and audit trace contract is tracked in +[`EVENTS_AND_AUDIT.md`](EVENTS_AND_AUDIT.md). +Policy decision, source provenance, and explain-response contracts are tracked +in [`POLICY_CONTRACTS.md`](POLICY_CONTRACTS.md). +The experimental remote WebUI bundle loading design is tracked in +[`REMOTE_WEBUI_BUNDLES.md`](REMOTE_WEBUI_BUNDLES.md). ## Layer Model @@ -35,14 +42,15 @@ The kernel must not own product semantics such as users, tenants, RBAC decisions ## Current Compatibility Responsibilities During the staged split, `govoplan-core` still contains compatibility surfaces -for access, auth, tenancy, RBAC, governance, audit, CSRF/API helpers, and -secret helpers. The extracted access implementation now lives in -`govoplan-access`; live legacy ORM table definitions have been split across -their platform owners while retaining historical table names. The old core -model, route, admin-service, and access-security import shims have been -removed; callers must use module-owned imports or kernel capabilities. The -remaining compatibility surfaces are temporary until the matching platform -modules are fully self-contained: +for tenancy settings, governance/policy contracts, audit helpers, CSRF/API +helpers, and secret helpers. The extracted access implementation lives in +`govoplan-access`; live ORM table definitions have been split across their +platform owners using module-prefixed table names. The old core route, +admin-service, and access-security re-export modules have been removed. +Callers must use module-owned imports, the public `govoplan_access.auth` request +dependency API, or kernel capabilities. +The remaining platform compatibility surfaces are temporary until the matching +platform modules are fully self-contained: - `govoplan-access` - `govoplan-tenancy` @@ -54,6 +62,18 @@ New code should avoid deepening these compatibility dependencies. Prefer explici Core must not import module feature pages or module business logic directly. It should interact with modules through manifests, entry points, metadata, capabilities, events, and route contributions. +The compatibility/deprecation plan for the current split line is: + +- keep documented public compatibility imports until the owning module exposes a + stable replacement and all in-tree callers have migrated +- remove deep implementation re-export modules once callers can use module-owned + public APIs or kernel capabilities +- preserve migration/table compatibility for already-created development and + release databases +- document remaining compatibility surfaces here and in the owning module README +- reject new cross-module imports that bypass manifests, capabilities, events, + or public module APIs + ## Stable Kernel Contracts The following contracts are the baseline API that modules can rely on: @@ -71,9 +91,15 @@ The following contracts are the baseline API that modules can rely on: - WebUI module contribution contract - navigation metadata contract - command/event envelope contract +- policy decision and source provenance contract in `govoplan_core.core.policy` Changes to these contracts must be versioned or accompanied by compatibility shims. +This list is the Milestone A kernel-contract freeze baseline. New module work +may extend the kernel by adding explicit contracts, but existing contracts must +remain source-compatible through the 0.1.x split line unless a migration shim +and deprecation note are provided. + Known access-related capability names are defined in `govoplan_core.core.access`, including: @@ -99,22 +125,23 @@ access/tenant ORM models when they need labels, group membership, default access provisioning, counts, audit actor labels, or tenant metadata. FastAPI route dependencies for authenticated endpoints are access-owned and -published from `govoplan_access.backend.auth.dependencies`. Routers may import -that dependency module directly until a more generic request-principal adapter -exists; they must not import access ORM models or other access implementation -internals. +published from `govoplan_access.auth`. Routers may import that public API for +`ApiPrincipal`, `get_api_principal`, `has_scope`, `require_scope`, and +`require_any_scope`; they must not import access ORM models or +`govoplan_access.backend.*` implementation internals. Current live table ownership: -- `govoplan-tenancy`: `tenants` -- `govoplan-access`: `accounts`, `users`, `groups`, `roles`, - `system_role_assignments`, `user_group_memberships`, - `user_role_assignments`, `group_role_assignments`, `api_keys`, - `auth_sessions` -- `govoplan-admin`: `governance_templates`, - `governance_template_assignments` +- `govoplan-tenancy`: `tenancy_tenants` +- `govoplan-access`: `access_accounts`, `access_users`, `access_groups`, + `access_roles`, `access_system_role_assignments`, + `access_user_group_memberships`, `access_user_role_assignments`, + `access_group_role_assignments`, `access_api_keys`, + `access_auth_sessions` +- `govoplan-admin`: `admin_governance_templates`, + `admin_governance_template_assignments` - `govoplan-audit`: `audit_log` -- `govoplan-core`: `system_settings` +- `govoplan-core`: `core_system_settings` Current admin route ownership follows the same boundary: access contributes users, groups, roles, system accounts/roles, auth, sessions, and API-key @@ -143,6 +170,183 @@ campaign file-share access, and core retention can call campaign-owned cleanup logic without importing campaign ORM models. Keep these contracts small DTO/protocol surfaces and register concrete behavior from the owning module. +## API Efficiency Contracts + +GovOPlaN uses conditional GET and delta collections to reduce reload cost +without giving every module a custom synchronization format. + +### Conditional GET + +Core applies conditional GET handling centrally for successful JSON `GET` +responses: + +- Responses receive a weak `ETag` based on the serialized JSON body. +- Responses are marked `Cache-Control: private, no-cache`. +- Responses vary by `Authorization`, `Cookie`, `X-API-Key`, and + `Accept-Language`. +- Matching `If-None-Match` requests return `304 Not Modified` without a body. +- Responses with `Set-Cookie`, `Content-Disposition`, `Content-Encoding`, a + non-JSON content type, a non-200 status, or `Cache-Control: no-store` are not + converted. + +The WebUI `apiFetch` client keeps an in-memory conditional cache for reusable +safe requests. It sends `If-None-Match` after an endpoint has returned an ETag, +returns the cached payload on `304`, and clears the cache generation after +unsafe methods. + +This avoids retransmitting unchanged snapshots. It does not identify which row +changed inside a collection. + +### Delta Collections + +Collection endpoints that can expose row-level changes should use the shared +delta contract instead of inventing module-specific formats. + +Core provides `core_change_sequence` as the shared monotonic change sequence. +Modules record append-only entries in the same database transaction as the +resource write. Watermarks are encoded as `seq:` and should be treated +as opaque by clients. + +Backend shape: + +```json +{ + "items": [], + "deleted": [], + "watermark": "opaque-next-watermark", + "has_more": false, + "full": false +} +``` + +Fields: + +- `items`: changed or current items since the requested watermark. +- `deleted`: deleted item markers with at least `id`, and optionally + `resource_type`, `revision`, and `deleted_at`. +- `watermark`: opaque value the client sends as `since` on the next request. +- `has_more`: true when the client should request the next page with the + returned watermark. +- `full`: true when the response is a full snapshot rather than an incremental + delta. + +Section-level settings endpoints use the same contract but replace `items` +with: + +- `item`: the full settings object when `full: true`. +- `sections`: a map of changed settings sections when `full: false`. +- `changed_sections`: ordered section identifiers the client can merge into its + local settings object. + +Recommended query parameters: + +- `since`: opaque previous watermark. If omitted or expired, return a full + snapshot with `full: true`. +- `limit`: maximum number of changed items plus deleted markers. +- `include_deleted`: whether deleted markers should be returned. +- `cursor`: opaque keyset cursor for table pages where offset shifts would make + row-level merging unsafe. + +Modules should record changes with: + +- `module_id`: the owning module, for example `files`. +- `collection`: the delta collection, for example `files.assets`. +- `resource_type`: stable row kind, for example `file` or `folder`. +- `resource_id`: stable resource identifier. +- `operation`: `created`, `updated`, or `deleted`. +- `tenant_id`: tenant scope when the change is tenant-owned. +- `payload`: small, non-secret routing metadata that helps determine whether a + tombstone belongs to the requested view. + +Sequence retention is explicit. Cleanup jobs must call +`prune_sequence_entries(...)` rather than deleting `core_change_sequence` rows +directly. Pruning records a retention floor per module, collection, and tenant +scope. Endpoints compare incoming watermarks with that floor; a watermark older +than the floor is not safe for incremental replay, so the endpoint must return a +full snapshot with `full: true`. A first-use `seq:0` watermark remains valid +until such a floor exists, even if unrelated collections have advanced the +global sequence. + +### Cursor/Keyset Pages + +Offset pagination remains supported for compatibility and for first page loads, +but it is not safe as the merge anchor for row-level deltas on page 2 and later. +When a delta-capable table can be paged beyond the first page, the endpoint +should expose keyset cursors: + +- Core provides `encode_keyset_cursor`, `decode_keyset_cursor`, and + `keyset_query_fingerprint` in `govoplan_core.core.pagination`. +- Cursors are opaque to clients and contain the endpoint scope, query + fingerprint, and last-row keyset values. +- The fingerprint must include every query input that changes membership or + order: scope, tenant, page size, sort column, sort direction, and filters. +- Reusing a cursor with different sort or filter parameters must fail with a + client error rather than returning a mismatched slice. +- Responses may still include `page`, `page_size`, `pages`, and `total` for + existing UI components, but `cursor` identifies the current slice and + `next_cursor` is the safe anchor for the next slice. +- The first visit to an arbitrary page can use offset compatibility. The + response should include the start cursor for that page so later reloads and + delta requests use keyset semantics. + +Concrete consumers: + +- `GET /api/v1/files/delta`: without `since`, returns the current files/folders + snapshot for the requested owner/campaign scope. With `since=seq:`, + returns changed files, changed folders, and tombstones for resources that + left the current view. +- `GET /api/v1/campaigns/delta`: returns accessible campaign rows and campaign + tombstones when ownership, sharing, or soft deletion removes a campaign from + the current list. +- `GET /api/v1/campaigns/{campaign_id}/workspace/delta`: returns a workspace + snapshot first, then changed campaign/version metadata and optional summary + refreshes when version, job, issue, or delivery-attempt changes invalidate the + workspace view. +- `GET /api/v1/campaigns/{campaign_id}/jobs/delta`: returns a paginated job + table snapshot first, then stable row deltas for cursor-backed job pages. + The job list also supports offset compatibility for first visits to a page and + returns `cursor`/`next_cursor` for stable reloads. Filtered, created, deleted, + or stale-watermark requests fall back to a full page snapshot when pagination + membership can shift. +- `GET /api/v1/admin/users/delta`, `/groups/delta`, `/roles/delta`, + `/system/roles/delta`, `/system/accounts/delta`, and `/api-keys/delta`: + return access administration row deltas with tombstones where rows leave the + visible view. +- `GET /api/v1/admin/system/settings/delta`: returns section deltas for system + defaults, tenant capability flags, language packages, privacy retention + policy, maintenance mode, and raw settings. +- `GET /api/v1/admin/tenant/settings/delta`: returns tenant-local setting + sections and also reports language-section changes when system language + packages or enabled language codes change. +- `GET /api/v1/admin/configuration-changes/delta`: returns changed + configuration requests and history records. +- `GET /api/v1/admin/audit` and `/api/v1/admin/audit/delta`: return append-only + audit events using the same scope, sort, and filter query parameters. The list + supports offset compatibility plus `cursor`/`next_cursor` keyset paging; the + delta endpoint can replay changes against a cursor-backed slice. +- `GET /api/v1/mail/settings/delta`: returns mail profile row deltas plus the + current scoped mail profile policy when profile-policy dependencies changed. + The WebUI consumes this for system, tenant, user, group, and campaign mail + settings panels. +- `GET /api/v1/files/connectors/settings/delta`: returns file connector + profile, credential, connector-space, and scoped connector-policy deltas. + Credential changes also include referencing profiles because profile rows + display credential-derived state. + +Open retrofit scope: + +- Additional module-specific settings pages should expose section deltas as + their settings APIs stabilize. Remaining likely candidates are future + booking/resource configuration pages and settings pages introduced by new + modules. +- Remaining high-volume tables should adopt the cursor/keyset contract before + enabling arbitrary-page row deltas. Current rollout follow-ups: + `govoplan-files#22` for large file-space server windows, + `govoplan-mail#9` for provider-aware mailbox message cursors, + `govoplan-calendar#7` for event-window deltas, + `govoplan-tenancy#1` for tenant administration row deltas, and + `govoplan-admin#2` for governance/module-operation list deltas. + ## Module Responsibilities A module owns one bounded feature area. A module can include both backend and WebUI code in the same repository so feature behavior and frontend integration evolve together. @@ -195,6 +399,18 @@ NavItem( ) ``` +Core validates manifest shape when the platform registry is built. The current +supported manifest contract version is `1`, and frontend asset manifests use +contract version `1`. Registry validation rejects unsupported contract versions, +invalid module ids, duplicate dependency declarations, self-dependencies, +mismatched migration/frontend metadata, invalid frontend package names, and +frontend/nav routes that do not declare usable paths and labels. + +Backend route contributions are also validated before they are mounted. Startup +routers and live module activation fail fast if two routers register the same +HTTP method and path. That keeps OpenAPI output and FastAPI route order from +silently masking a module collision. + ## Database And Migrations Core owns the database/session lifecycle. Modules access the database through core session dependencies and register their models/migrations through their manifest. @@ -209,6 +425,10 @@ Rules: - Optional module migrations may create multiple Alembic heads. Verification should compare the database heads to the configured script heads instead of assuming one linear revision when multiple modules are enabled. +- Treat migrations as release artifacts. Unreleased migrations may be squashed + or rewritten before a stable release; released revision IDs are immutable + once an installation may have recorded them. Each stable release records its + public migration heads in `docs/migration-release-baselines.json`. ## Install, Uninstall, And Catalogs @@ -290,6 +510,31 @@ capabilities with `usePlatformUiCapabilities("admin.sections")`, filters them by `anyOf`/`allOf`, and renders them without importing the contributing module's components directly. +The configurable dashboard follows the same pattern. Core contributes only a +minimal `/dashboard` fallback when no `dashboard` WebUI module is active. The +`govoplan-dashboard` module owns the real `/dashboard` route and collects +widgets exposed through the `dashboard.widgets` capability: + +```ts +const dashboardWidgets: DashboardWidgetsUiCapability = { + widgets: [ + { + id: "ops.health", + title: "Operations health", + moduleId: "ops", + defaultSize: "wide", + anyOf: ["ops:operations:read"], + render: ({ settings, refreshKey }) => createElement(OpsHealthWidget, { settings, refreshKey }) + } + ] +}; +``` + +Dashboard widgets are module contributions, not cross-module imports. A widget +may render components from its own module and core components only. The +dashboard module is responsible for layout, visibility, refresh context, and +future server-side layout persistence. + ## Icon Rules Icons are resolved centrally by core. @@ -354,14 +599,221 @@ The repository includes `scripts/check_dependency_boundaries.py`. It enforces th - access source may not import files/mail/campaign internals - feature modules may not import access implementation internals - feature modules may not add new direct imports of sibling feature modules -- FastAPI routers may import the published - `govoplan_access.backend.auth.dependencies` dependency API +- FastAPI routers may import the published `govoplan_access.auth` dependency API - the transitional allowlist is expected to stay empty Any future exception is extraction debt and must be temporary, documented in the script with a reason, and removed when a capability/API/event contract replaces it. +## Boundary Decision Register + +These durable decisions close older exploratory core issues. Implementation +work should live in the owning module repositories once a boundary is clear. + +Decision principles: + +- Prefer connector-first when an external specialist system is likely to remain + the system of record. +- Create a native module only when GovOPlaN must own domain semantics, + permissions, audit, retention, configuration-package fragments, or workflow + state. +- Keep optional behavior behind core-mediated capabilities, events, DTOs, route + contributions, and UI contribution points. +- Do not create repositories just because a possible product area exists. + +### Templates And Reporting + +Tracking: `govoplan-core#190`, `govoplan-templates#1`, +`govoplan-reporting#1`. + +Decision: templates and reporting are separate modules. + +`govoplan-templates` owns: + +- reusable renderable templates for letters, permits, emails, forms, reports, + certificates, and notices +- template versioning, merge-field declarations, rendering profiles, output + format choices, and preview contracts +- template package fragments that other modules can reference + +`govoplan-reporting` owns: + +- report definitions, data selection, dashboards, BI views, scheduled outputs, + and export targets +- report permissions, report execution history, generated report evidence, and + report-specific retention inputs +- downstream export handoff to files, dataflow, connectors, or publication + surfaces + +Boundary: + +- Templates do not own data selection, aggregation, scheduling, or BI semantics. +- Reporting may call template rendering through a capability when a formatted + report output is needed. +- Campaign, mail, files, workflow, and cases use templates/reporting through + capabilities and DTOs, never direct imports. + +### Sources, RSS, Datasources, And Dataflow + +Tracking: `govoplan-core#192`, `govoplan-core#197`, +`govoplan-core#198`, `govoplan-connectors#3`, +`govoplan-connectors#4`. + +Decision: do not create `govoplan-datasources` or `govoplan-dataflow` until a +first executable use case proves that connector/reporting/workflow ownership is +too narrow. + +First slice: + +- `govoplan-connectors` owns RSS/Atom consume/emit connector profiles, + connector health, external references, source lifecycle metadata, and + source/publish capability boundaries. +- `govoplan-files` owns file-backed governed locations and uploaded/stored file + evidence. +- `govoplan-reporting` owns report/data views and scheduled outputs. +- `govoplan-workflow` owns process state, approvals, scheduling of process + steps, and human review. + +Future `govoplan-datasources` is justified when GovOPlaN needs a broad source +catalogue for SQL databases, CSV/Excel files, APIs, RSS feeds, uploaded files, +and governed file locations with shared ownership, credentials, schema +discovery, refresh cadence, provenance, and permission boundaries. + +Future `govoplan-dataflow` is justified when GovOPlaN needs first-class +pipelines for ingestion, transformation, validation, scheduling, lineage, +publication, audit events, reruns, and source-to-source workflows. + +Monthly extraction/transformation work should start as a configuration package +and module collaboration across connectors, files, workflow, reporting, and +possibly templates. Create datasources/dataflow repositories only after that +package exposes repeated contracts that do not belong to an existing module. + +### Calendar, Scheduling, And Appointments + +Tracking: `govoplan-core#193`, `govoplan-calendar#1`, +`govoplan-calendar#2`, `govoplan-scheduling#1`, +`govoplan-appointments#1`. + +Decision: use three separate modules. + +`govoplan-calendar` owns: + +- calendar collections, events, recurrence, availability/free-busy, resources, + iCalendar import/export, CalDAV/Open-Xchange-style calendar adapters, and + calendar WebUI surfaces + +`govoplan-scheduling` owns: + +- Terminfindung, meeting-time polls, participant availability collection, + candidate-slot ranking, conflict explanations, reminders, and the handoff + from a selected slot to calendar/appointment/workflow modules + +`govoplan-appointments` owns: + +- Terminbuchung/fixed-slot appointment booking, appointment types, booking + rules, capacity, cancellation/no-show state, public/internal booking flows, + and appointment evidence + +Boundary: + +- Calendar provides time primitives and external calendar integration. +- Scheduling chooses a suitable time. +- Appointments owns booked appointment workflows and public/internal booking + semantics. +- Mail and notifications deliver invitations/reminders through capabilities. + +### Forms And Workflow Handoff + +Tracking: `govoplan-core#194`, `govoplan-forms#1`. + +Decision: forms are a reusable module boundary, with runtime behavior separated +from workflow semantics. + +`govoplan-forms` owns: + +- form definitions, schemas, validation rules, field visibility rules, + localization, versioning, admin editing, and reusable form package fragments + +`govoplan-forms-runtime` owns, when implemented: + +- public/internal submissions, drafts, submitted values, validation evidence, + attachment references, submission receipts, and handoff events + +Boundary: + +- Forms do not own cases, workflow transitions, tasks, or portal identity. +- Workflow/cases consume form submission events and evidence references. +- Files owns uploaded file storage and file permissions. +- Reporting/dataflow may consume submitted data through governed DTOs or + source lifecycle contracts. + +### OpenDesk Integration Profile + +Tracking: `govoplan-core#195`, `govoplan-connectors#5`, +`govoplan-idm#1`, `govoplan-mail#5`, `govoplan-calendar#2`, +`govoplan-connectors#1`. + +Decision: OpenDesk is an integration profile, not a monolithic module. + +Ownership: + +- identity: `govoplan-idm` plus `govoplan-access` +- mail/groupware: `govoplan-mail` +- calendar: `govoplan-calendar` +- files/documents: `govoplan-files` and later `govoplan-dms` +- projects/tasks: `govoplan-connectors` OpenProject connector first +- inventory/health/profile diagnostics: `govoplan-connectors` + +The OpenDesk profile should describe required connector profiles, shared +identity assumptions, health checks, and optional module combinations. It must +not create direct module-to-module imports. + +### Project Management And OpenProject + +Tracking: `govoplan-core#196`, `govoplan-connectors#1`. + +Decision: connector-first. Do not create a native `govoplan-projects` module +yet. + +OpenProject integration belongs in `govoplan-connectors` first: + +- profile test +- project and work-package lookup +- external-reference storage +- selected publish/synchronize capabilities for tasks, workflow, or cases + +A native project module is justified only if GovOPlaN needs to own project +semantics beyond cases, tasks, workflow, appointments, documents, and reporting, +for example portfolios, project budgets, project-level resource planning, or +governed project records that cannot remain in OpenProject. + +### Public-Sector Integration Landscape + +Tracking: `govoplan-core#186`, `govoplan-core#215`, +`govoplan-connectors#2`, `govoplan-connectors#3`. + +Decision: core owns strategy and routing; connectors owns executable +integration catalogue entries and operator inventory. + +Core documents: + +- product-level integration strategy +- native-vs-connector decisions +- owning module routing +- roadmap sequencing + +`govoplan-connectors` owns: + +- connector entry schema +- external system catalogue +- connector profiles and diagnostics +- source consume/publish lifecycle +- external references + +When a target needs executable behavior, create the implementation issue in the +owning module repository and keep only cross-module decisions in core. + ## Module Lifecycle Core exposes the installed module catalog through the admin API and WebUI. The @@ -565,6 +1017,8 @@ directory with: - `GOVOPLAN_INSTALLER_RUN_DIR` - `GOVOPLAN_DATABASE_URL` +- `GOVOPLAN_DATABASE_URL_PGTOOLS` for PostgreSQL URLs converted to the + `postgresql://` form expected by `pg_dump`, `pg_restore`, and `psql` - `GOVOPLAN_DATABASE_BACKUP_PATH` - `GOVOPLAN_DATABASE_BACKUP_METADATA` @@ -631,10 +1085,20 @@ Backend verification from core: ```bash cd /mnt/DATA/git/govoplan-core -./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign +./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-dashboard/src/govoplan_dashboard ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign ./.venv/bin/python scripts/check_dependency_boundaries.py ``` +`scripts/check-focused.sh` runs npm with an isolated temporary npm user config +so developer-local npm settings do not create release-check warning noise. + +Focused module contract and permutation verification: + +```bash +cd /mnt/DATA/git/govoplan-core +bash scripts/check-module-matrix.sh +``` + Core WebUI host verification: ```bash diff --git a/docs/POLICY_CONTRACTS.md b/docs/POLICY_CONTRACTS.md new file mode 100644 index 0000000..04fab8b --- /dev/null +++ b/docs/POLICY_CONTRACTS.md @@ -0,0 +1,105 @@ +# GovOPlaN Policy Contracts + +GovOPlaN has several policy families that are moving out of core into owning +modules. The shared kernel contract keeps their decision and provenance shape +consistent while each module still owns its domain rules. + +## Current Policy Inventory + +| Policy area | Current owner | Runtime surface | Notes | +| --- | --- | --- | --- | +| Privacy retention | `govoplan-policy` routes with compatibility helpers in core | `/api/v1/admin/privacy-retention/policies/{scope}` and `/explain` | System, tenant, user, group, and campaign sources merge into the effective retention policy. Parent locks block lower-level widening. | +| Mail profile policy | `govoplan-mail` | `/api/v1/mail/policies/{scope}` | Uses the same source-step path format for system, tenant, owner, and campaign provenance. | +| RBAC/access policy | `govoplan-access` | access capabilities in `govoplan_core.core.access` | Permission decisions should use access capability contracts. Explain responses should adopt `PolicyDecision` when an API-level explanation is added. | +| Governance defaults | `govoplan-admin` plus `govoplan-access` materializer | admin settings, governance template routes, access materialization capability | System governance can block tenant-local groups, roles, and API keys. | +| Delegation and ownership policy | access/campaign/mail/files modules | capability checks and owner-scoped APIs | Source provenance should use this contract when policies become externally explainable. | + +## Policy Decision + +The shared DTO lives in `govoplan_core.core.policy.PolicyDecision`. + +```json +{ + "allowed": false, + "reason": "Parent retention policy locks lower-level changes.", + "source_path": [ + { + "scope_type": "system", + "scope_id": null, + "path": "system", + "label": "System", + "applied_fields": ["allow_lower_level_limits"], + "policy": {} + } + ], + "requirements": ["raw_campaign_json_retention_days"], + "details": { + "blocked_fields": ["raw_campaign_json_retention_days"] + } +} +``` + +`allowed` is the effective answer for the checked action. `reason` is a stable, +human-readable summary. `source_path` lists the policy sources that explain the +answer. `requirements` lists machine-readable blockers or prerequisites, and +`details` carries domain-specific structured context. + +Every source step should be concrete enough for an operator to understand the +decision without knowing internal merge rules. Use real scope labels such as +`System`, `Tenant`, `Owner user`, `Group`, or a campaign/profile name. Include +the stable `path`, the fields applied by that step, and the local policy +fragment that caused them. This lets UIs render explanations like +`System: Allow > Tenant: Deny without override` without additional lookups. +If a policy family cannot expose the full local fragment for security reasons, +it must still include a redacted structured value that identifies the applied +field and the effective allow/deny or lock state. + +## Source Path Format + +Policy source paths are stable string identifiers for provenance steps: + +- `system` +- `:` + +Supported scope types are `system`, `tenant`, `user`, `group`, and `campaign`. +Examples: + +- `tenant:4a45b4fe-1d86-43ce-9d10-6022333f4d4b` +- `campaign:campaign%2Fwith%20space` + +Use `policy_source_path()` and `parse_policy_source_path()` instead of building +or splitting these strings manually. + +## Retention Explain Endpoint + +`GET /api/v1/admin/privacy-retention/policies/{scope_type}/explain` returns: + +- `scope_type` and optional `scope_id` +- `decision`, using the shared `PolicyDecision` shape +- `effective_policy` +- optional `parent_policy` +- `effective_policy_sources` +- `parent_policy_sources` +- `blocked_fields` + +The endpoint is read-only. Enforcement remains in the existing policy write +path. For lower-level scopes, `blocked_fields` is derived from the parent +policy's `allow_lower_level_limits`; clients can use it to disable local +controls before attempting a write. + +## Frontend Contract + +Policy UIs must: + +- render effective source provenance when `effective_policy_sources` is present +- display a field-level path when the source data is shown next to a specific + setting, using concrete source labels and stop at the first non-overridable + deny/lock +- disable local field controls when the parent policy sets that field's + lower-level limit to `false` +- avoid sending locked fields or re-enable attempts in save payloads +- show inherited values separately from local overrides + +The core WebUI helper `privacyRetentionParentAllowsField()` centralizes the +field-lock decision used by the retention editor and its lightweight module +tests. diff --git a/docs/POSTBOX_E2EE_ARCHITECTURE.md b/docs/POSTBOX_E2EE_ARCHITECTURE.md new file mode 100644 index 0000000..00850b5 --- /dev/null +++ b/docs/POSTBOX_E2EE_ARCHITECTURE.md @@ -0,0 +1,134 @@ +# Postbox End-To-End Encryption Architecture + +This document records the strategic encryption target for GovOPlaN postboxes. +It does not require the first postbox implementation to ship full E2EE, but it +defines the architecture so early data models and APIs do not make the stronger +model impossible. + +The core principle is that a postbox can become a trusted administrative +communication channel without requiring the server to see plaintext content. +The server may route, store, authorize, audit, retain, and expire messages while +message bodies and attachments remain client-encrypted. + +## Goals + +- asynchronous encrypted delivery for internal and portal-facing postboxes +- personal, organizational, role-bound, and function-bound postboxes +- attachments encrypted with the message +- access based on current role/function membership when configured +- honest retraction and expiry semantics +- auditable key access, delivery, and fetch events +- support for external recipients without platform accounts +- replaceable identity and trust providers + +## Envelope Model + +The target model is envelope encryption: + +- Generate one random data encryption key per message or attachment set. +- Encrypt content with an authenticated encryption algorithm. +- Wrap the data encryption key for each authorized recipient or role mailbox. +- Store only ciphertext, wrapped keys, signed manifests, and governed metadata + on the server. + +Algorithm choices should remain replaceable behind a crypto profile. The first +profile should prefer standard, reviewed primitives such as HPKE for key +wrapping and AEAD encryption for content. + +## Identity And Device Keys + +The platform should distinguish: + +- account identity +- tenant membership +- role/function assignment +- device key +- postbox binding + +Identity providers and directories can authenticate users and provide membership +facts, but they must not see postbox private keys or message plaintext. + +The trust layer should provide: + +- public key directory +- account or identity signing keys +- per-device encryption keys +- device registration and revocation +- key rotation and epoch tracking +- recovery policy hooks + +## Role And Function Postboxes + +Role-bound access needs special handling. A postbox can be bound to an +organizational unit and a role or function. Current members can access current +messages according to policy; former members should lose access to not-yet +fetched material when revocation is still technically enforceable. + +The target design should support role encryption keys or an equivalent +rewrapping service: + +- sender encrypts the content key for the role/function postbox +- access service verifies current membership and required assurance +- trust service rewraps the content key to the actor's current device key +- audit records the key access decision and fetch event + +Key epochs are required when role membership changes. Older messages may remain +readable according to policy, but new access must use the current epoch. + +## External Recipients + +External recipients may need one-time or time-limited access without a full +platform account. The target model should support capability links or invitation +tokens that are: + +- scoped to specific message or attachment resources +- time-limited +- optionally one-time +- protected by an out-of-band secret, passphrase, or stronger external identity + proof +- revocable before key fetch +- fully audited + +## Retraction Semantics + +GovOPlaN should be honest about retraction. + +Before a recipient fetches a key or decrypts content, the system can revoke +tokens, remove wrapped-key access, expire links, and delete ciphertext according +to retention policy. + +After a recipient has decrypted or copied plaintext, the system cannot make the +recipient forget it. The platform can only record access, revoke future access, +notify parties, and apply legal or organizational controls. + +The UI must explain this distinction whenever it offers expiry, retraction, or +message withdrawal. + +## Server Responsibilities + +The server remains important even when content is encrypted: + +- store ciphertext and signed manifests +- store routing and policy metadata +- enforce access before key release or rewrapping +- provide public key directory access +- emit notifications without plaintext content +- record audit events +- enforce retention and expiry where possible +- expose diagnostics for delivery and key-access failures + +## Module Ownership + +- `govoplan-postbox` owns postbox bindings, postbox messages, message metadata, + and postbox UI. +- `govoplan-identity-trust` owns device keys, public key directory, key epochs, + and assurance integration. +- `govoplan-access` owns current identity, membership, function, delegation, and + permission decisions. +- `govoplan-policy` owns retention, retraction, and security policy decisions. +- `govoplan-audit` owns durable audit traces. +- `govoplan-files` owns managed file storage when encrypted postbox attachments + are backed by file objects. + +No module should import another module's internals to decrypt content. All +interaction must use capabilities, DTOs, and audited service contracts. diff --git a/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md b/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md new file mode 100644 index 0000000..3b5dc96 --- /dev/null +++ b/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md @@ -0,0 +1,276 @@ +# Public-Sector Integration Strategy + +GovOPlaN should integrate with the existing public-sector software landscape +before deciding to replace specialist workflows. This document is the core +strategy index. The executable connector catalogue lives in +`govoplan-connectors/docs/PUBLIC_SECTOR_INTEGRATION_CATALOGUE.md`. + +## Strategy Labels + +Use one or more of these labels for every external system family: + +- `integrate`: GovOPlaN talks to the system through a stable API/protocol. +- `link`: GovOPlaN stores external references and opens the external system for + source-of-truth work. +- `import`: GovOPlaN consumes data or files into governed module storage. +- `synchronize`: GovOPlaN keeps selected records aligned both ways or through a + source-of-truth rule. +- `replace selected workflow`: GovOPlaN may own a narrow workflow where the + external product is weak, but does not replace the whole product family. +- `no first-class support`: GovOPlaN only stores manual references unless a + deployment project creates a specific connector. + +## Initial Classification + +| System family | Examples | Default strategy | Likely owner | +| --- | --- | --- | --- | +| File providers | SMB/CIFS, WebDAV, Nextcloud, Seafile, SFTP, S3 | integrate, import, link | `govoplan-files`, connector inventory in `govoplan-connectors` | +| Project/task management | OpenProject, Jira, Redmine, Microsoft Planner | link, synchronize selected records, replace selected workflow only after proof | `govoplan-connectors`, later `govoplan-tasks` or workflow modules | +| Identity providers | LDAP, Active Directory, OIDC, SAML, OpenDesk IDM | integrate, synchronize | `govoplan-idm`, `govoplan-access` | +| Mail and groupware | IMAP/SMTP, Open-Xchange, Exchange/M365, CalDAV/CardDAV | integrate, link | `govoplan-mail`, `govoplan-calendar`, `govoplan-connectors` | +| DMS/e-file/archive | d.velop/d.3, enaio, ELO, Fabasoft, CMIS, VIS/eAkte | link, import, synchronize selected metadata | `govoplan-dms`, `govoplan-files`, `govoplan-records` | +| ERP/finance/procurement | SAP, MACH, Infoma, DATEV, procurement feeds | export, import, synchronize; do not replace by default | `govoplan-erp`, `govoplan-procurement`, `govoplan-ledger`, `govoplan-payments` | +| Public-sector transport | FIT-Connect, XTA/OSCI, Peppol access points | integrate, publish, receive | dedicated protocol modules plus `govoplan-connectors` inventory | +| Standards registries | XRepository, XOE/V catalogues | link, import metadata/cache | `govoplan-connectors`, `govoplan-xoev` | +| Publication/data exchange | RSS, open-data APIs, API feeds, CSV/Excel drops | consume, publish, transform | proposed `govoplan-datasources`, proposed `govoplan-dataflow`, `govoplan-reporting` | +| Collaboration suites | Matrix, Jitsi, BigBlueButton, Nextcloud Talk, Collabora/OnlyOffice | integrate, link; native behavior only for governed evidence | `govoplan-connectors`, `govoplan-dms`, `govoplan-workflow` | +| Specialist Fachverfahren | register-specific and domain-specific systems | link first; integrate/import when a real project supplies contracts | domain module or deployment-specific connector | + +## Landscape Catalogue + +This catalogue is intentionally implementation-oriented. Each entry records the +first API/auth/data assumptions needed to turn an inventory entry into a +connector or module issue. + +### Citizen And Service Portals + +- Strategy: integrate/link first; replace selected intake workflow only when a + GovOPlaN portal package owns the complete journey. +- Protocol/API surface: REST/JSON APIs, form submission webhooks, OIDC/SAML + login, eID interfaces where available, file-upload callbacks, case-status + callbacks. +- Auth model: OIDC/SAML service clients, signed webhook secrets, tenant-scoped + API keys, later eID/trust-provider handoff. +- Data shape: applicant identity reference, application form payload, + attachment references, consent declarations, status events, receipt IDs. +- Deployment assumptions: externally reachable HTTPS, reverse proxy, portal DMZ + separation, strict CSRF/origin settings, large upload path. +- Risks: personal data exposure, duplicate identity mapping, partial + submissions, upload malware, inconsistent portal status models. +- MVP test path: submit a test application with one file, create a form + submission/case/task stub, return a receipt and status reference. +- Owner/priority: `govoplan-portal`, `govoplan-forms-runtime`, + `govoplan-files`, Wave 1. + +### DMS, E-File, Records, And Archives + +- Strategy: link/import/synchronize selected metadata; do not replace the DMS by + default. +- Protocol/API surface: CMIS, WebDAV, vendor REST APIs, S3/object archive + staging, file-plan export/import, archive handoff APIs. +- Auth model: service accounts, OAuth/OIDC where supported, mTLS for regulated + archives, secret references for vendor tokens. +- Data shape: document ID, version, file-plan/classification code, retention + metadata, owner/case reference, external URL, checksum, lock/legal-hold state. +- Deployment assumptions: usually internal network or VPN, strict storage + quotas, existing retention policies, archive immutability requirements. +- Risks: record duplication, broken legal hold, permission mismatch, version + drift, destructive retention/export mistakes. +- MVP test path: create a connector inventory entry, test read-only metadata + lookup, link one GovOPlaN file/case evidence item to an external document. +- Owner/priority: `govoplan-dms`, `govoplan-records`, `govoplan-files`, + `govoplan-connectors`, Wave 2/5. + +### ERP, Finance, Procurement, And Accounting + +- Strategy: export/import/synchronize selected records; replacement only by + narrow domain decision. +- Protocol/API surface: vendor REST/SOAP APIs, CSV/XML batch exchange, SFTP, + XRechnung/Peppol, XBestellung/procurement feeds, payment reconciliation files. +- Auth model: service accounts, client certificates, mTLS, SFTP keys, token + references, environment-specific account separation. +- Data shape: debtor/creditor reference, payment request, invoice, order, + budget/cost-center code, booking status, receipt/evidence reference. +- Deployment assumptions: batch windows, finance-system approval workflows, + test tenants often separated from production by vendor process. +- Risks: financial posting errors, double export, tax/legal data retention, + inconsistent master data, irreversible accounting handoff. +- MVP test path: dry-run export of one payment/accounting handoff file with + checksum, validation report, and no remote posting. +- Owner/priority: `govoplan-payments`, `govoplan-ledger`, + `govoplan-xrechnung`, `govoplan-erp`, `govoplan-procurement`, Wave 1/6. + +### Identity, IAM, And Directory Services + +- Strategy: integrate/synchronize; access remains GovOPlaN's local + authorization boundary. +- Protocol/API surface: LDAP, Active Directory, SCIM, OIDC, SAML, OpenDesk IDM + APIs, group membership sync, account deactivation feeds. +- Auth model: bind accounts, service clients, OIDC/SAML metadata, SCIM tokens, + certificate-backed clients where required. +- Data shape: account, user, group, membership, role claim, tenant/org-unit + mapping, status, external directory ID. +- Deployment assumptions: directory is usually internal; identity provider may + be organization-wide and not GovOPlaN-owned. +- Risks: privilege escalation through group mapping, stale memberships, account + collision, deprovisioning latency, tenant-boundary mistakes. +- MVP test path: read-only directory profile test, map one external group to a + tenant group, show a dry-run membership diff. +- Owner/priority: `govoplan-idm`, `govoplan-access`, Wave 1. + +### Groupware, Mail, Calendar, And Collaboration + +- Strategy: integrate/link; native behavior only where GovOPlaN needs governed + evidence or process state. +- Protocol/API surface: IMAP/SMTP, CalDAV/CardDAV, Open-Xchange APIs, Microsoft + Graph/EWS, Matrix APIs, Jitsi/BigBlueButton APIs, Collabora/OnlyOffice + integration points. +- Auth model: service accounts, delegated OAuth/OIDC, app passwords, mailbox + credentials, groupware-specific tokens, secret references. +- Data shape: mailbox folder/message references, event/free-busy data, meeting + URL, chat room ID, participant list, document-editing session reference. +- Deployment assumptions: often internal/existing tenant infrastructure; mail + and calendar may be separate from identity even in OpenDesk-style stacks. +- Risks: mail credential exposure, calendar privacy, double invitations, room + booking conflicts, chat/document data escaping retention rules. +- MVP test path: profile test for mailbox/calendar reachability, read-only + folder/free-busy lookup, create a non-production event/message draft. +- Owner/priority: `govoplan-mail`, `govoplan-calendar`, + `govoplan-connectors`, Wave 1/2. + +### Payment And Public Cashier Systems + +- Strategy: integrate/export/import; keep the payment provider or cashier as + source of settlement truth. +- Protocol/API surface: payment provider APIs, redirect/callback flows, + reconciliation files, SEPA/export formats, cash-register/cashier interfaces. +- Auth model: provider API keys, signed webhooks, client certificates, mTLS, + tenant-specific merchant accounts. +- Data shape: payment intent, amount/currency, payer reference, provider + transaction ID, settlement status, receipt, refund/cancellation reference. +- Deployment assumptions: public callback URLs, strict environment separation, + PCI-sensitive providers, finance reconciliation cadence. +- Risks: duplicate charges, callback replay, amount mismatch, refund workflow + gaps, evidence-retention mistakes. +- MVP test path: sandbox payment intent, signed callback verification, receipt + evidence link, reconciliation dry-run. +- Owner/priority: `govoplan-payments`, `govoplan-ledger`, Wave 1/6. + +### Reporting, BI, Open Data, And Publication + +- Strategy: consume/publish/transform; native reporting owns GovOPlaN views, not + every external BI product. +- Protocol/API surface: SQL read replicas, CSV/Excel export/import, REST APIs, + RSS/Atom, open-data APIs, SFTP/WebDAV publication targets. +- Auth model: read-only DB users, API tokens, SFTP keys, OAuth clients, public + anonymous publication profiles where appropriate. +- Data shape: dataset metadata, schema/version, report parameters, generated + file references, publication URL, freshness/lineage, validation results. +- Deployment assumptions: publication can be public or internal; generated + datasets need retention and provenance. +- Risks: leaking restricted data, stale publications, schema drift, expensive + queries, untraceable manual transformations. +- MVP test path: publish one report/export as a governed file plus RSS/Atom + entry with checksum, timestamp, and permission check. +- Owner/priority: `govoplan-reporting`, `govoplan-connectors`, possible future + `govoplan-datasources`/`govoplan-dataflow`, Wave 2. + +### Public-Sector Protocols And Registries + +- Strategy: integrate/publish/receive; protocol modules own protocol semantics. +- Protocol/API surface: FIT-Connect, XTA/OSCI, XRepository, XOE/V, XRechnung, + XBestellung, Peppol, registry-specific Fachverfahren APIs. +- Auth model: certificates, mTLS, service accounts, destination credentials, + protocol-specific trust anchors and key rotation. +- Data shape: transport envelope, payload schema/version, destination IDs, + receipt/acknowledgement, message status, standard-specific metadata. +- Deployment assumptions: regulated trust chains, test/prod endpoint + separation, formal onboarding, strict logging and retention expectations. +- Risks: invalid schemas, failed delivery receipts, certificate expiry, wrong + destination routing, protocol version drift. +- MVP test path: validate a sample payload against a schema, test endpoint + reachability in sandbox, store receipt/evidence reference. +- Owner/priority: `govoplan-fit-connect`, `govoplan-xoev`, + `govoplan-xrechnung`, `govoplan-xta-osci`, `govoplan-connectors`, Wave 1/2. + +### File Providers And Shared Storage + +- Strategy: integrate/import/link; files module owns GovOPlaN file semantics. +- Protocol/API surface: SMB/CIFS, WebDAV, Nextcloud, Seafile, SFTP, S3, + local/object storage profiles. +- Auth model: service accounts, user credentials, app tokens, OAuth where + supported, secret references, environment variables for deployment-managed + credentials. +- Data shape: file ID/path, provider object ID, checksum, MIME type, size, + version/ETag, owner, permission snapshot, imported file reference. +- Deployment assumptions: internal networks, large files, existing shares, + variable permissions, provider-specific rate limits. +- Risks: permission mismatch, stale imports, overwrites, duplicate files, path + traversal, storage growth. +- MVP test path: profile test, list folder, import one file into governed + storage, keep provider reference and checksum. +- Owner/priority: `govoplan-files`, `govoplan-connectors`, Wave 0/1. + +### Project, Task, And Case-Adjacent Systems + +- Strategy: connector-first for OpenProject/Jira/Redmine; native module only + when GovOPlaN owns project semantics. +- Protocol/API surface: OpenProject API v3, webhooks, Jira/Redmine REST APIs, + Microsoft Graph for Planner/Project where applicable. +- Auth model: API tokens, OAuth/OIDC apps, webhook secrets, service accounts. +- Data shape: project ID, work package/task ID, status, assignee reference, + external URL, version/lock token, publish/sync trace. +- Deployment assumptions: external project tool remains source of truth for + broad project management; GovOPlaN links selected records. +- Risks: task duplication, bidirectional sync conflicts, permission mismatch, + over-mirroring comments/attachments. +- MVP test path: OpenProject profile test, project/work-package lookup, + external-reference round-trip. +- Owner/priority: `govoplan-connectors`, later `govoplan-tasks`/workflow/cases + consumers, Wave 0/2. + +### Specialist Fachverfahren + +- Strategy: link first; integrate/import only when a deployment project supplies + concrete contracts and a domain owner. +- Protocol/API surface: vendor APIs, CSV/XML batch imports, SFTP, database + views, message queues, protocol-specific transports. +- Auth model: usually service accounts, VPN, mTLS, SFTP keys, or vendor tokens. +- Data shape: domain-specific record IDs, status, applicant/person references, + file/evidence references, case/status events. +- Deployment assumptions: strongly local/vendor-specific, often no stable test + API, data model differs by jurisdiction. +- Risks: brittle vendor contracts, legal source-of-truth ambiguity, high + customization cost, migration expectations. +- MVP test path: inventory entry and manual external-reference link; require a + project-specific connector issue before automation. +- Owner/priority: domain module or deployment-specific connector, case by case. + +## Prioritization Rules + +1. Start with connectors that unblock Wave 0 or Wave 1 reference journeys. +2. Prefer open standards and self-hosted/open-source APIs where they are common + in public-sector deployments. +3. Treat inventory-only entries as useful because operators need a map of their + software landscape even before automation exists. +4. Keep connector code in the owning connector/protocol module. Domain modules + consume capabilities, DTOs, external references, and events through core. +5. Every executable connector needs health diagnostics, secret-reference + handling, lifecycle state, audit events, and retirement behavior. + +## Connector Catalogue Handoff + +`govoplan-connectors` owns the detailed catalogue entry shape: + +- connector type key +- category and owner module +- supported directions and trigger modes +- credential and secret handling +- health check and diagnostics payload +- external-reference shape +- required capabilities and optional module combinations +- lifecycle support + +Core should only keep strategy, routing, and cross-module architecture notes. +Connector implementation and public-sector target inventory belong in +`govoplan-connectors`. diff --git a/docs/RBAC_MANIFEST.md b/docs/RBAC_MANIFEST.md deleted file mode 100644 index c31d632..0000000 --- a/docs/RBAC_MANIFEST.md +++ /dev/null @@ -1,291 +0,0 @@ -# Multi Seal Mail - Current RBAC and Resource-Access Model - -**Updated:** 2026-06-16 -**Current migration head:** `f5a6b7c8d9e0` - -## Authorization Equation - -An operation is permitted only when every applicable layer allows it: - -```text -effective role/API-key capability -AND resource ownership/share access -AND workflow state -AND active governance/policy constraints -``` - -RBAC answers what an actor may do. ACLs answer which resource the actor may do it to. Workflow state and policy decide whether the operation is currently valid. - -## Identity and Scope - -```text -Account global login identity -+- User membership tenant-local identity - +- direct tenant roles - +- active group memberships - | +- inherited tenant roles - +- tenant-local API keys - -Account -+- direct system-role assignments -``` - -A browser session has one active tenant membership. System privileges do not silently grant tenant data access. API keys remain tenant-local and receive the intersection of their configured scopes and their owner's live tenant scopes on every request. - -## Wildcards - -```text -tenant:* every canonical tenant permission -system:* every canonical system permission -* legacy alias interpreted as tenant:* only -``` - -Tenant wildcards never grant system permissions. - -## Canonical Tenant Permissions - 53 - -### Campaigns - -```text -campaign:read -campaign:create -campaign:update -campaign:copy -campaign:archive -campaign:delete -campaign:share -campaign:validate -campaign:build -campaign:review -campaign:send_test -campaign:queue -campaign:control -campaign:send -campaign:retry -campaign:reconcile -``` - -### Recipients - -```text -recipients:read -recipients:write -recipients:import -recipients:export -``` - -### Files - -```text -files:read -files:download -files:upload -files:organize -files:share -files:delete -files:admin -``` - -### Reports and Audit - -```text -reports:read -reports:export -reports:send -audit:read -``` - -### Mail Servers - -```text -mail_servers:read -mail_servers:use -mail_servers:test -mail_servers:write -mail_servers:manage_credentials -``` - -### Tenant Administration - -```text -admin:users:read -admin:users:create -admin:users:update -admin:users:suspend - -admin:groups:read -admin:groups:write -admin:groups:manage_members - -admin:roles:read -admin:roles:write -admin:roles:assign - -admin:api_keys:read -admin:api_keys:create -admin:api_keys:revoke - -admin:settings:read -admin:settings:write -admin:policies:read -admin:policies:write -``` - -## Canonical System Permissions - 18 - -```text -system:tenants:read -system:tenants:create -system:tenants:update -system:tenants:suspend - -system:accounts:read -system:accounts:create -system:accounts:update -system:accounts:suspend - -system:roles:read -system:roles:write -system:roles:assign - -system:access:read -system:access:assign - -system:audit:read -system:settings:read -system:settings:write -system:governance:read -system:governance:write -``` - -`system:access:*` remains as a compatibility/read and assignment boundary for cross-tenant/system access handling. It is not a separate primary UI area. - -## Default Tenant Roles - -- **Owner:** `tenant:*`. At least one active operational owner must remain. -- **Tenant administrator:** settings, policies, users, groups, roles and API keys plus read access to campaigns/files/reports/audit. Real delivery remains separately delegable. -- **Administrator (legacy):** all tenant permissions for upgraded installations. -- **Access administrator:** membership and assignment management within delegation limits. -- **Campaign manager:** prepare, validate and build campaigns; no review approval or real delivery by default. -- **Reviewer:** inspect and approve prepared campaign messages. -- **Sender:** mock-test, queue, control, send, retry and reconcile prepared campaigns; can use/test approved mail profiles. -- **File manager:** managed file operations without campaign delivery rights. -- **Viewer:** read campaigns, recipients, files and reports. -- **Auditor:** read campaigns, recipient evidence, reports and audit records; export detailed evidence. - -## Default System Roles - -- **System owner:** `system:*`, protected. At least one active account must retain it. -- **System administrator:** all specific system permissions, editable and not protected. -- **System auditor:** read-only system registry/settings/governance/audit role, editable. - -## Delegation Ceiling - -For role definition, assignment and API-key creation: - -```text -requested scopes subset of actor delegateable scopes -``` - -Rules: - -1. Tenant roles may contain tenant scopes only. -2. System roles may contain system scopes only. -3. Definition rights and assignment rights are separate. -4. Group definition and group membership management are separate. -5. API-key scopes are intersected with the owner's current effective scopes on every request. -6. Suspended accounts, users, tenants or groups stop contributing access immediately. -7. Administrative updates are field-sensitive; a user with only status authority cannot change role assignments. - -## Campaign Ownership and ACLs - -A campaign has exactly one owner: - -```text -owner user OR owner group -``` - -Additional active shares may target users or groups with `read` or `write`. - -Resolution: - -- owner user: read and write; -- member of owner group: read and write; -- explicit read share: read; -- explicit write share: read and write; -- `tenant:*`: tenant-wide ACL bypass; -- ordinary campaign permission without ownership/share: no object access. - -ACLs do not add capabilities. A write share still needs the specific permission for update, validation, review, send, report, retry or reconciliation. - -## Sensitive Recipient Boundary - -Recipient-complete campaign JSON, message data and job detail require `recipients:read`. Recipient edits require `recipients:write`; exports require `recipients:export`; import is reserved for the dedicated recipient import/list workflow. - -## Files - -| Permission | Operations | -|---|---| -| `files:read` | list, search, inspect, resolve metadata | -| `files:download` | download file bytes and generated ZIP archives | -| `files:upload` | upload files and ZIP contents | -| `files:organize` | create folders, rename, move, copy and bulk rename | -| `files:share` | create/revoke file shares | -| `files:delete` | delete/hide files and folders subject to retention | -| `files:admin` | tenant-wide administration of user/group file spaces | - -## Mail Servers - -| Permission | Boundary | -|---|---| -| `mail_servers:read` | profile metadata and effective policy visibility | -| `mail_servers:use` | select an approved profile without reading secrets | -| `mail_servers:test` | run server-side connection tests | -| `mail_servers:write` | define/edit profiles in allowed scopes | -| `mail_servers:manage_credentials` | create/replace SMTP/IMAP secrets or campaign-level credentials where policy allows | - -Reusable encrypted profiles now exist. Effective usability is also constrained by hierarchical mail-profile policy, ownership, allowed/forced profile sets, credential inheritance mode, the lower-level override switch for that mode, and allow/deny patterns. - -## Sessions, API Keys and CSRF - -- Browser login creates an HttpOnly session cookie and a separate readable CSRF cookie. -- Unsafe cookie-authenticated requests require matching CSRF cookie/header and stored CSRF hash. -- API keys remain supported for CLI/automation and do not use browser CSRF. -- Login responses still expose a compatibility session token in the response body; the WebUI does not persist it. - -## Legacy Compatibility - -Runtime aliases remain only for names that are no longer canonical, including: - -```text -campaign:write -attachments:read -attachments:write -admin:users -admin:users:write -admin:api_keys:write -admin:settings -system:tenants:write -system:access:write -``` - -Canonical scopes are not widened by runtime alias expansion after migration. - -## Deferred Permission Families - -Add these only with their corresponding implemented features: - -```text -templates:* -address_books:* -recipient_lists:* -connectors:* -dsar:* -system:monitoring:read -system:backups:run -system:backups:restore -system:updates:apply -system:updates:rollback -``` - -A separate `retention:*` family is not currently canonical because retention is managed through system settings and tenant policy scopes. Add it only if retention operation duties need separation from general policy/settings administration. diff --git a/docs/RELEASE_CATALOG_WORKFLOW.md b/docs/RELEASE_CATALOG_WORKFLOW.md deleted file mode 100644 index f02479a..0000000 --- a/docs/RELEASE_CATALOG_WORKFLOW.md +++ /dev/null @@ -1,129 +0,0 @@ -# Release Catalog Workflow - -GovOPlaN release catalogs are published by `govoplan-web` as static JSON and -verified by `govoplan-core` before installer plans are accepted. - -Private signing keys must stay outside all git repositories. Public keyrings -are published with the website. - -## One-Time Key Setup - -Create the first catalog signing key on the release machine: - -```bash -cd /mnt/DATA/git/govoplan-core -KEY_DIR="$HOME/.config/govoplan/release-keys" -mkdir -p "$KEY_DIR" -./.venv/bin/python scripts/generate-catalog-keypair.py \ - --key-id release-key-1 \ - --private-key "$KEY_DIR/release-key-1.pem" \ - --public-key "$KEY_DIR/release-key-1.pub" \ - --keyring "$KEY_DIR/catalog-keyring.json" -``` - -Keep `release-key-1.pem` private. The generated keyring contains only public -material. - -## Publish Current Release Catalog - -Generate the signed catalog into `govoplan-web`: - -```bash -cd /mnt/DATA/git/govoplan-core -KEY_DIR="$HOME/.config/govoplan/release-keys" -scripts/publish-release-catalog.sh \ - --version 0.1.4 \ - --sequence 202607071340 \ - --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ - --build-web -``` - -This writes: - -- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json` -- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json` - -The wrapper validates the catalog with core using the generated public keyring. - -## Publish After A New Release - -For normal module/core releases, first tag and push the module/core repos. Then -publish the website catalog: - -```bash -cd /mnt/DATA/git/govoplan-core -KEY_DIR="$HOME/.config/govoplan/release-keys" -scripts/publish-release-catalog.sh \ - --version \ - --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ - --build-web \ - --commit \ - --tag \ - --push -``` - -The website tag is `catalog-v`. The public URL is: - -```text -https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json -``` - -The public keyring URL is: - -```text -https://govoplan.add-ideas.de/catalogs/v1/keyring.json -``` - -## Integrated Release Script - -`scripts/push-release-tag.sh` can publish the web catalog after module and core -tags have been pushed: - -```bash -cd /mnt/DATA/git/govoplan-core -KEY_DIR="$HOME/.config/govoplan/release-keys" -scripts/push-release-tag.sh \ - --bump subversion \ - --publish-web-catalog \ - --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ - --build-web-catalog -``` - -Use `--catalog-signing-key` more than once during a key rotation window. The -catalog will contain multiple signatures and the public keyring will include the -corresponding public keys. - -## Consumer Configuration - -On a GovOPlaN installation that should consume the official stable catalog: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true -GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable -GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true -``` - -For production, copy the public keyring into deployment configuration and pin it -locally. Do not rely on a URL-fetched keyring as the only trust root. - -## Core Updates - -`stable.json` includes a top-level `core_release` section for operator/update -tooling. Core is intentionally not listed as a normal module entry, because it -must not be added to saved enabled-module state. Core upgrades should remain an -operator-supervised package update with restart and health checks. - -## Key Rotation - -1. Generate the next private key outside git. -2. Run `publish-release-catalog.sh` with both signing keys. -3. Publish the web catalog/keyring. -4. Roll the new public keyring into installations. -5. Stop signing with the old key after the supported fleet trusts the new key. -6. Mark compromised keys as revoked in the public keyring and publish a higher - sequence catalog signed by a trusted uncompromised key. - diff --git a/docs/RELEASE_DEPENDENCIES.md b/docs/RELEASE_DEPENDENCIES.md index f3b4094..06f8cfe 100644 --- a/docs/RELEASE_DEPENDENCIES.md +++ b/docs/RELEASE_DEPENDENCIES.md @@ -1,8 +1,17 @@ # GovOPlaN Release Dependencies -Release installs must not depend on sibling checkout paths. Local development can keep editable installs and `file:` WebUI links, but release packaging should resolve modules from tagged git refs or from a package registry. +This document owns release package composition, signed package catalogs, +license checks, catalog publishing, migration baselines, and the final release +checklist. -## Backend +Operator runtime configuration and module install/uninstall execution live in +`DEPLOYMENT_OPERATOR_GUIDE.md`. + +## Backend Packages + +Release installs must not depend on sibling checkout paths. Local development +can keep editable installs and `file:` WebUI links, but release packaging must +resolve modules from tagged git refs or from a package registry. Local development: @@ -18,280 +27,41 @@ cd /mnt/DATA/git/govoplan-core ./.venv/bin/python -m pip install -r requirements-release.txt ``` -`.[server]` is resolved relative to the current working directory. If you create the virtualenv elsewhere, still run the install command from the core checkout: +`.[server]` is resolved relative to the current working directory. If you +create the virtualenv elsewhere, still run the install command from the core +checkout: ```bash cd /mnt/DATA/git/govoplan-core /tmp/govoplan-release-test/bin/python -m pip install -r requirements-release.txt ``` -`requirements-release.txt` pins the module repositories to the release tag. Update those refs when cutting a release: +`requirements-release.txt` pins the module repositories to the release tag. +Update those refs when cutting a release: ```text -govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.4 -govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.4 -govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.4 -govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.4 -govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.4 -govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.4 -govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.4 -govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.4 -govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.4 +govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.6 +govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.6 +govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.6 +govoplan-organizations git@git.add-ideas.de:add-ideas/govoplan-organizations.git v0.1.6 +govoplan-identity git@git.add-ideas.de:add-ideas/govoplan-identity.git v0.1.6 +govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.6 +govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.6 +govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.6 +govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.6 +govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.6 +govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.6 ``` -## Runtime Module Package Changes +## WebUI Packages -The admin module manager can hot-enable and hot-disable packages that are -already installed. It does not install or uninstall Python/npm packages from -inside the running server. +Local development uses `webui/package.json`, which may point at sibling module +checkouts while active development is happening. -For runtime package changes, create an operator install plan in Admin > System > -Modules. The module manager shows the trusted installer preflight status and -blocks unsafe uninstalls before the operator touches packages. - -Preflight from the server shell: - -```bash -govoplan-module-installer --format shell -``` - -Apply from a controlled operator shell while maintenance mode is active: - -```bash -govoplan-module-installer --apply --build-webui -``` - -For real install/uninstall work, prefer supervised mode with the deployment's -restart command and health endpoint: - -```bash -govoplan-module-installer \ - --supervise \ - --migrate \ - --build-webui \ - --health-url http://127.0.0.1:8000/health \ - --restart-command '' -``` - -To let the admin UI trigger package work without executing pip/npm inside a -FastAPI request, run the installer daemon in a separate operator shell. This is -the preferred development/early-production mode for now because the operator can -watch output, stop before queueing risky changes, and keep restart commands -deployment-specific: - -```bash -govoplan-module-installer \ - --daemon \ - --migrate \ - --build-webui \ - --database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ - --database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \ - --database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ - --health-url http://127.0.0.1:8000/health \ - --restart-command '' -``` - -Admin > System > Modules can then queue the saved install plan as a supervised -request. Install rows can be planned directly from the approved package catalog; -uninstall rows are generated from installed, disabled modules so the Python -distribution and WebUI package names do not need to be typed by hand. The -daemon claims one queued request at a time and writes request/run records below -`runtime/module-installer`. For process-manager one-shot usage or tests, use -`--daemon-once`. The daemon also writes -`runtime/module-installer/daemon.status.json`; check it with: - -```bash -govoplan-module-installer --daemon-status --format json -``` - -The installer uses a runtime lock, snapshots `pip freeze` plus WebUI -`package.json`/`package-lock.json`, writes a run record below -`runtime/module-installer/runs`, and marks planned rows as applied only after -all commands succeed. When `--migrate` is used with a `sqlite:///` database URL, -the installer also snapshots the SQLite database with SQLite's backup API before -running migrations. For other database engines, pass external backup/restore -hooks: - -```bash -govoplan-module-installer \ - --supervise \ - --migrate \ - --database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ - --database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \ - --database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ - --health-url http://127.0.0.1:8000/health \ - --restart-command '' -``` - -The backup command runs before migrations. The restore-check command validates -the produced backup artifact before migrations proceed, without restoring over -the live database. The restore command is stored in the run record and runs -during rollback unless an override is passed to `--rollback`. - -Supervised mode treats package command failure, migration failure, restart -failure, and health timeout as rollback triggers. It restores the Python/WebUI -package snapshots, re-runs the restart command when supplied, and restores the -saved install plan state so the operator can correct it. The supervisor must -run outside the FastAPI server process; the admin UI saves and validates plans -but does not mutate packages from an HTTP request. - -After a successful install plan, the installer adds installed modules to saved -startup state by default so the restarted server can discover and enable them. -After a successful uninstall plan, the installer removes uninstalled modules -from saved startup state by default. Use -`--no-activate-installed-modules` or -`--keep-uninstalled-modules-in-desired` only for staged rollout workflows that -will update module state separately. - -Uninstall is non-destructive by default. A planned uninstall row can set -`destroy_data: true` to request destructive module retirement. The module must -provide an automated retirement provider, and the installer snapshots the -database before dropping module-owned tables. For SQLite this uses the built-in -snapshot path; for PostgreSQL or another non-SQLite database, provide -`--database-backup-command`, `--database-restore-check-command`, and -`--database-restore-command`. If a destructive run fails during package removal, -the installer restores the database snapshot before returning the failed run -result; supervised restart/health failures also roll back through the normal -supervisor path. - -Package rollback is automatic. SQLite database rollback is automatic for -installer runs that used `--migrate` and captured a database snapshot. -Non-SQLite rollback is automatic when the run used -`--database-backup-command` and `--database-restore-command`; otherwise migrated -non-SQLite runs are blocked before package changes are applied. - -Rollback uses the saved run snapshot: - -```bash -govoplan-module-installer --rollback -govoplan-module-installer --rollback --database-restore-command '' -``` - -Database hook commands run with these environment variables: - -- `GOVOPLAN_INSTALLER_RUN_DIR`: the run snapshot directory -- `GOVOPLAN_DATABASE_URL`: the configured database URL -- `GOVOPLAN_DATABASE_BACKUP_PATH`: a suggested backup artifact path inside the - run directory -- `GOVOPLAN_DATABASE_BACKUP_METADATA`: optional JSON metadata path that backup - commands may write for operator diagnostics - -Avoid embedding secrets directly in commands; prefer environment variables, -service credentials, or deployment-local secret injection. - -Inspect installer history and lock state from the operator shell: - -```bash -govoplan-module-installer --list-runs --format json -govoplan-module-installer --show-run --format json -govoplan-module-installer --lock-status --format json -govoplan-module-installer --list-requests --format json -govoplan-module-installer --show-request --format json -govoplan-module-installer --cancel-request --format json -govoplan-module-installer --retry-request --format json -``` - -Package catalogs can be local files or remote static resources, for example -served by `govoplan-web`. Set `GOVOPLAN_MODULE_PACKAGE_CATALOG` for a local file -or `GOVOPLAN_MODULE_PACKAGE_CATALOG_URL` for a remote catalog matching -`docs/module-package-catalog.example.json`; the admin UI will show those entries -and can save them into the install plan. This keeps the release approval -decision outside the running server while avoiding hand-typed package refs. -Remote catalogs can be cached for offline inspection: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json -``` - -Validate the catalog before handing it to operators: - -```bash -govoplan-module-installer --validate-package-catalog docs/module-package-catalog.example.json --format json -``` - -Release catalogs should be signed, channel-gated, expiring, and sequence -tracked. The supported signing format is an Ed25519 signature over the -canonical catalog JSON object with the `signature` and `signatures` fields -removed. Core accepts the legacy single `signature` field and the newer -`signatures` array used during key rotation. Sign a catalog with: - -```bash -govoplan-module-installer \ - --sign-package-catalog docs/module-package-catalog.example.json \ - --catalog-signing-key-id release-key-1 \ - --catalog-signing-private-key /path/to/ed25519-private.pem -``` - -Validate an approved release catalog with: - -```bash -govoplan-module-installer \ - --validate-package-catalog docs/module-package-catalog.example.json \ - --require-signed-catalog \ - --approved-catalog-channel stable \ - --catalog-trusted-key release-key-1= \ - --format json -``` - -For the admin UI/daemon path, configure the same policy through environment -variables: - -```bash -GOVOPLAN_MODULE_PACKAGE_CATALOG=/path/to/catalog.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true -GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable -GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":""}' -GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json -GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true -``` - -Trusted keys can also be loaded from -`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE`. A URL-backed keyring is -supported for development and tightly controlled deployments through -`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL` plus -`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE`, but production systems -should pin trusted keys locally. - -Catalog entries can declare `license_features`. If -`GOVOPLAN_LICENSE_ENFORCEMENT=true`, core blocks planning catalog installs -whose required features are not present in the configured offline license: - -```bash -GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json -GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json -GOVOPLAN_LICENSE_ENFORCEMENT=true -``` - -See `docs/CATALOG_TRUST_AND_LICENSING.md` for key rotation, replay protection, -and licensing details. See `docs/RELEASE_CATALOG_WORKFLOW.md` for the concrete -release-machine workflow that generates signing keys and publishes signed -catalogs through `govoplan-web`. Unsigned catalogs remain usable for local -development when signature enforcement is off, but the admin UI labels them as -unsigned. - -Install rows must use tagged package/git refs or registry packages, not local -`file:` or workspace links. The installer daemon can run `npm install` and -`npm run build` for WebUI package changes; that is the supported path. Browser -remote bundles are still experimental and should be treated as a controlled -deployment option, not the normal install/uninstall mechanism. - -Module manifests can declare core compatibility bounds and uninstall guard -providers. Preflight blocks incompatible manifest contracts/core versions, -active modules, desired startup state, protected modules, and active dependents. -Default uninstall is non-destructive: module data and schema remain dormant if -the package is removed. Persistent-data guards therefore warn by default instead -of requiring export/delete. A module that supports explicit data/schema -retirement should register a retirement provider. When `destroy_data` is set on -an uninstall plan row, that provider is allowed to destroy module-owned data -after the installer has captured a database snapshot; otherwise it is used only -for preflight reporting. - -## WebUI - -Local development uses `webui/package.json`, which may point at sibling module checkouts while active development is happening. - -Release WebUI installs should use `webui/package.release.json`. It points module dependencies at the same tagged git repositories. After the module tags referenced there exist, generate the committed release lockfile without touching the development package files: +Release WebUI installs should use `webui/package.release.json`. It points +module dependencies at the same tagged git repositories. After the module tags +referenced there exist, generate the committed release lockfile without +touching the development package files: ```bash cd /mnt/DATA/git/govoplan-core @@ -300,16 +70,48 @@ cd webui PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build ``` -The module repositories include root-level npm package manifests so git installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`, `@govoplan/files-webui`, `@govoplan/mail-webui`, `@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository roots even though their source lives below `webui/src`. +The module repositories include root-level npm package manifests so git +installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`, +`@govoplan/files-webui`, `@govoplan/mail-webui`, +`@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository +roots even though their source lives below `webui/src`. -The normal release path is automated by `scripts/push-release-tag.sh`: it bumps or accepts the target version, updates Python/WebUI/module manifest versions, commits/tags/pushes the module repositories first, regenerates `webui/package-lock.release.json`, and then commits/tags/pushes core. If the working tree has already been bumped, pass the current version explicitly: +### Release Lockfile Strategy + +The supported release composition currently is the full GovOPlaN product: core +plus access, admin, tenancy, organizations, identity, policy, audit, +dashboard, files, mail, campaign, calendar, docs, and ops. Keep one committed +full-product release lockfile at +`webui/package-lock.release.json`, generated from +`webui/package.release.json` in a clean release workspace. Development +`package-lock.json` may continue to point at local `file:` dependencies. + +Frontend module permutations are regression-tested through +`GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through +committed lockfiles for every possible combination. If a smaller composition +becomes a separately shipped product, add an explicit release manifest and +lockfile pair for that product, for example +`package.release.files-mail.json` and `package-lock.release.files-mail.json`, +generated in a clean release workspace from tagged git dependencies. + +## Release Tag Script + +The normal release path is automated by `scripts/push-release-tag.sh`: it bumps +or accepts the target version, updates Python/WebUI/module manifest versions, +commits/tags/pushes the module repositories first, regenerates +`webui/package-lock.release.json`, and then commits/tags/pushes core. If the +working tree has already been bumped, pass the current version explicitly: ```bash cd /mnt/DATA/git/govoplan-core -scripts/push-release-tag.sh --version 0.1.2 +scripts/push-release-tag.sh --version 0.1.6 ``` -The script also includes GovOPlaN roadmap/scaffold module repositories that do not yet have package metadata. Those repositories are committed, tagged, and pushed with the same release tag, but they are tag-only until they contain `pyproject.toml`, module manifests, or WebUI packages. Tag-only repositories are not listed in `requirements-release.txt` or `webui/package.release.json`. +The script also includes GovOPlaN roadmap/scaffold module repositories that do +not yet have package metadata. Those repositories are committed, tagged, and +pushed with the same release tag, but they are tag-only until they contain +`pyproject.toml`, module manifests, or WebUI packages. Tag-only repositories +are not listed in `requirements-release.txt` or `webui/package.release.json`. Current tag-only module repositories: @@ -325,7 +127,6 @@ Current tag-only module repositories: - `govoplan-idm` - `govoplan-ledger` - `govoplan-notifications` -- `govoplan-ops` - `govoplan-payments` - `govoplan-portal` - `govoplan-reporting` @@ -338,17 +139,486 @@ Current tag-only module repositories: - `govoplan-xrechnung` - `govoplan-xta-osci` -### Release lockfile strategy +## Catalog Trust And Licensing -The supported release composition currently is the full Multi Seal Mail product: core plus access, admin, tenancy, policy, audit, files, mail, campaign, and calendar. Keep one committed full-product release lockfile at `webui/package-lock.release.json`, generated from `webui/package.release.json` in a clean release workspace. Development `package-lock.json` may continue to point at local `file:` dependencies. +GovOPlaN module install and uninstall must remain operator-controlled. The +running server may plan and validate package changes, but package mutation is +performed by the separate installer daemon or an operator shell during +maintenance mode. -Frontend module permutations are regression-tested through `GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through committed lockfiles for every possible combination. If a smaller composition becomes a separately shipped product, add an explicit release manifest and lockfile pair for that product, for example `package.release.files-mail.json` and `package-lock.release.files-mail.json`, generated in a clean release workspace from tagged git dependencies. +`govoplan-web` is the public static distribution surface for official catalog +resources: + +- signed module package catalogs, grouped by release channel +- public catalog keyrings +- public license verification keyrings +- examples and operator-facing download paths + +`govoplan-core` is the verifier and orchestrator: + +- fetches a local or remote module catalog +- verifies catalog signatures against configured trusted keys +- enforces approved release channels +- rejects expired or not-yet-valid catalogs +- records accepted catalog sequence numbers for replay protection +- checks catalog entry license feature requirements before planning installs +- writes installer plans and request records + +Feature and platform modules own their package artifacts, manifests, migration +metadata, retirement providers, and optional lifecycle behavior. + +Core accepts either a local catalog file or a remote URL: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG=/srv/govoplan/catalogs/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json +``` + +If both file and URL are set, the URL wins. The cache is used when a remote +fetch fails, so an operator can still inspect the last known catalog. A cached +catalog must still pass signature, freshness, channel, and replay validation. + +An official catalog is a JSON object with: + +- `catalog_version` +- `channel` +- `sequence` +- `generated_at` +- `not_before` when delayed activation is needed +- `expires_at` +- `modules` +- `signatures` + +Each module entry can declare: + +- backend package name and pinned install reference +- WebUI package name and pinned install reference +- display metadata and tags +- `license_features`, the feature entitlements required to plan that install + +The signature is Ed25519 over canonical JSON with both `signature` and +`signatures` removed. Core accepts the legacy single `signature` field and the +new `signatures` array. + +Trusted catalog keys are configured locally: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":""}' +``` + +For development or tightly controlled deployments, a keyring can be read from a +URL and cached: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL=https://govoplan.example/catalogs/v1/keyring.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE=/srv/govoplan/runtime/catalog-cache/keyring.json +``` + +Production installations should pin the trusted keyring locally or ship it +through deployment configuration. Fetching trusted keys from the same public +origin as the catalog is convenient, but that origin must not become the only +trust root. + +## Dependency Audits + +Dependency vulnerability checks are documented in +[`DEPENDENCY_AUDITS.md`](DEPENDENCY_AUDITS.md). The local audit runner is: + +```bash +cd /mnt/DATA/git/govoplan-core +bash scripts/check-dependency-audits.sh +``` + +The Gitea workflow in `.gitea/workflows/dependency-audit.yml` runs the same +check against release dependency refs on pushes, pull requests, and a weekly +schedule. + +Keyring entries support: + +- `key_id` +- `public_key` or `public_key_base64` +- `status`: `active`, `next`, `retired`, `revoked`, or `disabled` +- `not_before` +- `not_after` + +Rotation process: + +1. Add the next public key to the local trusted keyring with status `next`. +2. Publish catalogs signed by both current and next keys. +3. Upgrade installations so the next key is locally trusted. +4. Promote the next key to `active`. +5. Retire the old key only after every supported installation trusts the new + key. +6. Mark a compromised key `revoked` and publish a higher sequence catalog + signed by an uncompromised key. + +Use replay state in production: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true +``` + +Core records the accepted sequence per channel after a catalog entry is planned +from the admin interface. With strict sequence enforcement, a previously +accepted sequence is rejected; without strict enforcement, only older sequences +are rejected. Catalogs should always expire. + +The sequence state file is operational state, not a trust root. Keep it on +persistent storage and include it in normal backups: + +```json +{ + "channels": { + "stable": { + "last_sequence": 42, + "accepted_at": "2026-07-07T12:00:00Z", + "key_id": "release-key-1", + "source": "https://govoplan.example/catalogs/v1/channels/stable.json" + } + } +} +``` + +If the file is lost, restore it from backup. If no backup exists, reconstruct +each channel from the highest sequence already accepted in installer run +records, release records, or the currently deployed module package set. Do not +lower `last_sequence` to make an older catalog pass; publish a new higher +sequence catalog when the accepted point is uncertain. + +If the file is corrupted, copy it aside for incident review, validate the +current signed catalog with channel and freshness enforcement, then rewrite the +state with the known accepted sequence. Keep +`GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true` and approved-channel +checks enabled during recovery. Temporarily disabling +`GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE` allows revalidating the same +sequence, but older sequences remain rejected once the reconstructed +`last_sequence` is in place. + +Approved channels are deployment policy: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts +GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true +``` + +The admin UI can display other catalog metadata, but core rejects catalogs from +unapproved channels when validation is configured. + +Catalog entries can require license features: + +```json +"license_features": ["module.mail", "support.standard"] +``` + +Core checks those requirements against an offline license file before allowing +the entry into the install plan. + +```bash +GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json +GOVOPLAN_LICENSE_ENFORCEMENT=true +GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json +``` + +License files are JSON objects with: + +- `license_id` +- `subject` +- `features` +- `valid_from` +- `valid_until` +- `signature` + +Issue or renew a license from an operator/release shell that has the Ed25519 +private key: + +```bash +govoplan-module-installer \ + --issue-license /srv/govoplan/license.json \ + --license-id customer-2026-07 \ + --license-subject "Example Municipality" \ + --license-feature module.mail \ + --license-feature support.standard \ + --license-valid-until 2027-07-31T23:59:59Z \ + --license-signing-key-id license-issuer-1 \ + --license-signing-private-key /srv/govoplan/secrets/license-issuer-1.pem \ + --format json +``` + +Validate an imported license without exposing secrets: + +```bash +govoplan-module-installer \ + --validate-license /srv/govoplan/license.json \ + --license-trusted-key license-issuer-1="" \ + --require-trusted-license \ + --license-required-feature module.mail \ + --format json +``` + +The CLI and admin module catalog panel report the license id, subject, +validity window, signing key id, signed/trusted state, available features, and +missing entitlements for the configured package catalog. They do not expose +private signing material. + +License enforcement can run in observe-only mode by leaving +`GOVOPLAN_LICENSE_ENFORCEMENT` unset. In that mode, missing or invalid license +data is surfaced as a warning but does not block planning. + +Renewal is an ordinary re-issuance with a new `license_id`, extended +`valid_until`, and the full intended feature set. Import the renewed JSON to +`GOVOPLAN_LICENSE_FILE`, keep the previous file for audit, and validate it +before setting enforcement. + +Revocation is handled through the trusted license keyring. Mark a compromised +or invalid issuer key as `revoked` or `disabled`, publish or deploy the updated +keyring, then reissue affected licenses with an active key. Installations that +run with `GOVOPLAN_LICENSE_ENFORCEMENT=true` reject licenses signed only by a +revoked key after the local keyring is updated. + +Emergency fallback is deliberately explicit. Operators can temporarily unset +`GOVOPLAN_LICENSE_ENFORCEMENT` to keep package planning observable while a +license or keyring is recovered. Record the change in the operational incident +log, keep catalog signature and channel enforcement enabled, and restore +license enforcement after a trusted renewal validates successfully. + +Licensing is intentionally separate from open-source code licensing. The +catalog/license mechanism can govern support channels, official release +eligibility, hosted update access, professional support, or commercial +entitlements without changing the source license of the repositories. + +Production-grade distribution still needs remote registry/git artifact +resolution before package-manager apply, a hardened catalog publishing pipeline +in `govoplan-web`, and automated key rotation and emergency revocation drills. + +## Release Catalog Publishing + +GovOPlaN release catalogs are published by `govoplan-web` as static JSON and +verified by `govoplan-core` before installer plans are accepted. Private signing +keys must stay outside all git repositories. Public keyrings are published with +the website. + +Create the first catalog signing key on the release machine: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +mkdir -p "$KEY_DIR" +./.venv/bin/python scripts/generate-catalog-keypair.py \ + --key-id release-key-1 \ + --private-key "$KEY_DIR/release-key-1.pem" \ + --public-key "$KEY_DIR/release-key-1.pub" \ + --keyring "$KEY_DIR/catalog-keyring.json" +``` + +Keep `release-key-1.pem` private. The generated keyring contains only public +material. + +Generate the signed catalog into `govoplan-web`: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +scripts/publish-release-catalog.sh \ + --version \ + --sequence 202607071340 \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ + --build-web +``` + +This writes: + +- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json` +- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json` + +The wrapper validates the catalog with core using the generated public keyring. + +For normal module/core releases, first audit and record migration baselines, +then tag and push the module/core repos. Finally publish the website catalog: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python scripts/release-migration-audit.py --strict +``` + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +scripts/publish-release-catalog.sh \ + --version \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ + --build-web \ + --commit \ + --tag \ + --push +``` + +The website tag is `catalog-v`. The public URL is: + +```text +https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json +``` + +The public keyring URL is: + +```text +https://govoplan.add-ideas.de/catalogs/v1/keyring.json +``` + +`scripts/push-release-tag.sh` can publish the web catalog after module and core +tags have been pushed. It runs the migration release audit in automatic mode: +warning-only before the first recorded migration baseline, strict after a +baseline exists. Add `--strict-migration-audit` when you want to force strict +mode explicitly: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +scripts/push-release-tag.sh \ + --bump subversion \ + --strict-migration-audit \ + --publish-web-catalog \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ + --build-web-catalog +``` + +Use `--catalog-signing-key` more than once during a key rotation window. The +catalog will contain multiple signatures and the public keyring will include the +corresponding public keys. + +On a GovOPlaN installation that should consume the official stable catalog: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true +GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true +``` + +For production, copy the public keyring into deployment configuration and pin it +locally. Do not rely on a URL-fetched keyring as the only trust root. + +`stable.json` includes a top-level `core_release` section for operator/update +tooling. Core is intentionally not listed as a normal module entry because it +must not be added to saved enabled-module state. Core upgrades should remain an +operator-supervised package update with restart and health checks. + +Key rotation for published catalogs: + +1. Generate the next private key outside git. +2. Run `publish-release-catalog.sh` with both signing keys. +3. Publish the web catalog/keyring. +4. Roll the new public keyring into installations. +5. Stop signing with the old key after the supported fleet trusts the new key. +6. Mark compromised keys as revoked in the public keyring and publish a higher + sequence catalog signed by a trusted uncompromised key. + +## PostgreSQL Release Check + +Release candidates should pass a disposable PostgreSQL migration and startup +smoke check before tagging or publishing catalogs. Start the local testbed, +then run the permutation check from the core checkout: + +```bash +cd /mnt/DATA/git/govoplan-core/dev/postgres +cp .env.example .env +docker compose --env-file .env up -d + +cd /mnt/DATA/git/govoplan-core +set -a +. dev/postgres/.env +set +a +./.venv/bin/python scripts/postgres-integration-check.py \ + --database-url "$GOVOPLAN_POSTGRES_DATABASE_URL" \ + --reset-schema +``` + +The script checks migrations and `/health` startup for core-only, files-only, +mail-only, campaign-only, campaign+files, campaign+mail, and full-product +module sets. `--reset-schema` is destructive and must only be used against a +throwaway database. + +## Migration Baselines + +Development migrations may be small and numerous while a feature is moving. +Before a stable release, unreleased migrations may be rewritten or squashed into +a release-level baseline or release-to-release upgrade migration. After a +release tag has shipped, released migration revision IDs are immutable. + +The release policy is: + +- unreleased migrations may be folded before release; +- released migrations are never rewritten or deleted; +- each stable release records the public migration head revisions in + `docs/migration-release-baselines.json`; +- fresh installations should apply release-level baselines/upgrades, not + unreleased create-then-rename churn; +- release-to-release schema changes should be folded into one reviewed + migration per migration owner where practical. + +Audit the current graph during release preparation: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python scripts/release-migration-audit.py +``` + +Generate the reviewed/manual squash checklist: + +```bash +./.venv/bin/python scripts/release-migration-audit.py --squash-plan +``` + +After the release migrations have been reviewed and the graph is final, record +the release baseline: + +```bash +./.venv/bin/python scripts/release-migration-audit.py --record-release +``` + +Use strict mode to verify that the current heads are recorded: + +```bash +./.venv/bin/python scripts/release-migration-audit.py --strict +``` + +`scripts/push-release-tag.sh` runs the audit by default in automatic mode: +non-strict while no release baseline exists, strict after the first baseline is +recorded. Pass `--warn-migration-audit` for an explicit non-strict audit, +`--strict-migration-audit` to force strict mode, or `--skip-migration-audit` +only for emergency/manual release work. + +Before the first stable release, fold the current development chain into the +first public baseline and record that baseline in +`docs/migration-release-baselines.json`. The tracking issue is +`add-ideas/govoplan-core#223`. + +## Related Operator Documents + +- `DEPLOYMENT_OPERATOR_GUIDE.md`: runtime environment, explicit migrations, + backup/restore commands, module installer daemon/supervisor operation, and + rollback drills. +- `REMOTE_WEBUI_BUNDLES.md`: experimental browser-loaded module bundles for + controlled deployments; normal releases use package builds. ## Release Checklist - Keep Python package versions, WebUI package versions, and git tags aligned. -- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign, calendar, and scaffold module repositories together. -- Update `requirements-release.txt` and `webui/package.release.json` when the release tag changes. -- Generate the committed full-product release lockfile from `package.release.json` with `scripts/generate-release-lock.sh`. -- Add separate release manifest/lockfile pairs only for module compositions that are shipped as their own products. +- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign, + calendar, and scaffold module repositories together. +- Update `requirements-release.txt` and `webui/package.release.json` when the + release tag changes. +- Generate the committed full-product release lockfile from + `package.release.json` with `scripts/generate-release-lock.sh`. +- Run `scripts/release-migration-audit.py --strict` after recording a release + baseline. +- Run the PostgreSQL release check against a disposable database. +- Publish the signed catalog through the release catalog publishing flow above. +- Add separate release manifest/lockfile pairs only for module compositions + that are shipped as their own products. - Do not commit local sibling paths into release manifests. diff --git a/docs/REMOTE_WEBUI_BUNDLES.md b/docs/REMOTE_WEBUI_BUNDLES.md new file mode 100644 index 0000000..ac7401e --- /dev/null +++ b/docs/REMOTE_WEBUI_BUNDLES.md @@ -0,0 +1,161 @@ +# Remote WebUI Bundle Loading + +GovOPlaN WebUI modules normally ship through the core WebUI package graph: +install a tagged npm/git dependency, run `npm install`, rebuild the shell, and +restart or reload the served assets. Remote WebUI bundles are an experimental +future path for controlled deployments where a backend-enabled module is not in +the local WebUI package graph and the operator wants the shell to load its +frontend without rebuilding core. + +This design defines the target guardrails. The current rebuild/reload path +remains the default production path. + +## Current Rebuild Path + +The supported release path is: + +1. Install or remove backend and WebUI package dependencies through the trusted + installer CLI/daemon. +2. Snapshot package state before mutation. +3. Run `npm install` and optionally `npm run build`. +4. Restart or reload the served WebUI assets. +5. Use installer rollback if package apply, restart, or health checks fail. + +Strengths: + +- package manager and lockfile semantics stay conventional +- CSP can stay strict because all code is served as built assets +- rollback restores package files and lockfiles +- local development uses sibling workspace dependencies naturally + +Costs: + +- frontend changes require a rebuild/reload +- hot enabling a module with frontend code is not possible in the running shell +- failed rebuilds happen at install time, not at lazy module-load time + +## Remote Bundle Path + +Remote loading is only for modules that are enabled by the backend and absent +from `virtual:govoplan-installed-modules`. The backend manifest exposes: + +- `FrontendModule.asset_manifest` +- `asset_manifest_integrity` +- `asset_manifest_signature` +- `asset_manifest_public_key_id` +- `asset_manifest_contract_version` + +The WebUI shell fetches the asset manifest, verifies manifest integrity and/or +signature, validates the manifest contract, fetches the entry bundle, verifies +the entry integrity, imports the bundle, validates the exported +`PlatformWebModule`, and applies backend metadata before registering routes, +navigation, and UI capabilities. + +Unsigned and unhashed manifests are skipped. Entries without integrity are +skipped. A module id mismatch is skipped. + +## Asset Manifest Contract + +Contract version `1`: + +```json +{ + "contractVersion": "1", + "moduleId": "files", + "entry": "./files-webui.remote.js", + "entryIntegrity": "SHA-256-", + "moduleExport": "default" +} +``` + +The backend manifest carries the manifest-level trust metadata. The remote +asset manifest carries the concrete entry URL and entry digest. + +## Compatibility Checks + +Before a remote bundle is accepted: + +- backend module must be enabled +- local WebUI module with the same id must be absent +- manifest contract must be supported by the shell +- manifest `moduleId`, exported module `id`, and backend module id must match +- backend metadata remains authoritative for label, version, dependencies, + nav, and runtime UI capability exposure +- future contract versions must declare required shell capabilities so old + shells fail closed + +Remote bundles must not broaden backend permissions. They can only contribute +routes/nav/capabilities that the backend metadata and existing permission +checks allow. + +## CSP + +The current implementation imports verified entry bytes through a blob URL. A +production CSP for this mode must explicitly allow: + +- `connect-src` for the approved asset-manifest and bundle origins +- `script-src` for `blob:` only when remote loading is enabled +- no `unsafe-inline` requirement for remote modules + +If an installation cannot allow `blob:` scripts, the follow-up implementation +should use signed same-origin module assets with static URLs instead of blob +imports. Remote loading must be disableable by configuration so strict-CSP +deployments can keep the rebuild path only. + +## Cache Invalidation + +The shell cache key is: + +```text +:: +``` + +Release catalogs should publish immutable manifest and entry URLs or change at +least one cache-key component on every release. Emergency rollback can point the +backend manifest at a previous immutable manifest URL or lower the enabled +module version after the backend package rollback. + +Browsers may still cache remote responses. Approved asset servers should use: + +- long cache lifetimes only for content-addressed immutable assets +- short cache lifetimes or explicit revalidation for channel/latest manifest + aliases +- `Cache-Control: no-store` for emergency override manifests + +## Rollback + +Remote WebUI rollback is metadata rollback, not package-manager rollback: + +- if the backend package install rolls back, the previous backend frontend + metadata returns +- if only remote assets are bad, publish a new manifest URL or revert the + backend/frontend metadata to the last known good manifest +- failed remote loading must degrade by omitting the remote module frontend, + not by breaking the shell + +Installer run records should eventually include remote asset manifest URLs, +integrity, signature key ids, and load-test results when a plan enables a +remote frontend. + +## Local And Development Behavior + +Local development should keep using workspace/file dependencies and Vite. Remote +loading is useful for integration testing release artifacts, not for day-to-day +module UI development. + +Development deployments may use unsigned catalogs and local asset servers only +when signature/integrity enforcement is intentionally disabled. The remote +loader itself still requires an integrity hash or a verifiable signature. + +## Follow-Up Slices + +1. Add a server-side remote-bundle policy flag and surface whether remote + loading is enabled in platform metadata. +2. Add CSP documentation/config generation for strict rebuild-only mode versus + controlled remote-bundle mode. +3. Add an installer/catalog preflight that validates remote WebUI asset + manifests and records verified identity in installer run records. +4. Add Playwright coverage for a signed test remote bundle, failed digest, bad + module id, cache-key refresh, and fallback when the module is unavailable. +5. Add release tooling to emit immutable remote asset manifests with digest, + signature, key id, and rollback metadata. diff --git a/docs/UI_UX_DECISION_LEDGER.md b/docs/UI_UX_DECISION_LEDGER.md new file mode 100644 index 0000000..094b630 --- /dev/null +++ b/docs/UI_UX_DECISION_LEDGER.md @@ -0,0 +1,216 @@ +# GovOPlaN UI/UX Decision Ledger + +This ledger records product UI/UX decisions that affect admin, settings, +configuration, connector, policy, and module-management surfaces. It is a +binding design reference: future implementation should follow these decisions +unless the decision is explicitly revised here and affected screens are updated +to match. + +Active tracking issue: `add-ideas/govoplan-core#225`. + +## Operating Rule + +GovOPlaN must expose advanced platform capability without presenting the user +with every option at once. Non-technical users should be able to complete common +workflows through guided, plain-language flows. Expert and diagnostic detail may +exist, but it must be deliberately layered. + +The ethical design doctrine in `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` is the +normative baseline for this ledger. A screen can be visually quiet and still be +ethically complete only when it preserves context, consequence, +contestability, responsibility, and traceability at the point of action. + +## Binding Decisions + +| ID | Decision | Status | Applies To | +| --- | --- | --- | --- | +| UX-001 | Use progressive disclosure by default. Common decisions stay visible; advanced or hazardous options live in collapsed panels, later wizard steps, or explicit advanced sections. | Accepted | Admin, settings, connector setup, policy editors, module operations | +| UX-002 | Raw JSON is not a primary configuration editor. Every normal configuration path needs typed controls, validation, and help text. JSON may be shown for import/export, diagnostics, or expert inspection only. | Accepted | All admin/configuration UIs | +| UX-003 | Prefer guided workflows over option dumps for setup and risky changes. Use wizards for connector setup, configuration package import, module install/uninstall, destructive actions, and policy changes with broad impact. | Accepted | Connectors, package import, module lifecycle, governance/policy | +| UX-004 | Discover technical values when possible. Ask for the smallest user-known input, then discover and prefill technical fields for review. | Accepted | File connectors, mail/groupware, public URLs, future external providers | +| UX-005 | Disabled actions and failed steps must explain why they are unavailable, who can fix them, and where to go next. Silent disabled states are not acceptable for primary actions. | Accepted | All primary actions | +| UX-006 | Explanations must be available but quiet. Use short inline text and expose richer explanations through help affordances, side panels, expandable sections, or review steps. | Accepted | All complex forms and flows | +| UX-007 | Creation/editing should prefer modals or focused step flows when it reduces page clutter. Overview and comparison screens remain full-page. | Accepted | Settings/admin surfaces | +| UX-008 | Similar concepts must use shared placement and components: server/credential/policy rows, problem lists, review steps, advanced panels, confirmation modals, and empty/error states. | Accepted | Core WebUI and module WebUIs | +| UX-009 | Preflight and diagnostics are product UX. Validation, policy, permission, dependency, and capability failures must be written for operators before exposing internal details. | Accepted | Installer, connectors, policy, package import | +| UX-010 | Context, decision, and consequence must be visible together for actions that affect rights, duties, records, money, communication, retention, external systems, or workflow state. | Accepted | Workflow, admin, portal, policy, connector, records, payments | +| UX-011 | Navigation is not consent. Route changes, panel switches, and passive selection must not execute consequential actions without an explicit action surface. | Accepted | All WebUI surfaces | +| UX-012 | Automated actions must remain inspectable. The UI must show the system actor, trigger, policy result, observed effects, and failure/manual-intervention state when automation changes administrative state. | Accepted | Workflow, automation, connectors, tasks, audit | +| UX-013 | Contestable decisions must expose provenance. Denials, locks, generated outputs, calculated defaults, policy decisions, access decisions, and retention decisions need a reachable source path. | Accepted | Policy, access, templates, workflow, retention, records | +| UX-014 | Retraction, expiry, undo, rollback, and delete controls must state the real limit of the operation. Corrective or future-only actions must not be described as if they undo already observed effects. | Accepted | Postbox, files, records, installer, workflow, payments | + +## Confirmed Implementation Decisions + +These decisions were accepted on 2026-07-09 for the first implementation slice. +They shape reusable components and screen structure. Future changes must revise +this section and update affected screens. + +### DUE-001: Primary Admin Configuration Shell + +Decision: admin/configuration surfaces should standardize on a two-zone layout. + +- Left or top area: searchable overview/list, status, and primary actions. +- Main area: selected item summary and common settings. +- Modal/wizard: create, connect, edit, test, review, and confirm actions. +- Collapsed advanced panels: rarely used technical fields. + +Use this for file connectors and mail servers first, then migrate policy, +retention, API keys, and module operations. + +### DUE-002: Wizard Step Model + +Decision: standard wizard steps and names use this baseline: + +1. Choose type or scope. +2. Enter essentials. +3. Discover or test. +4. Configure ownership and policy. +5. Review changes and blockers. +6. Save or submit for operator action. + +Not every wizard needs every step, but flows should use these names and order +where applicable. + +This applies to explicit assisted setup, onboarding, import, preflight, and +risky multi-step operations. It is not the default shape for ordinary create or +edit dialogs. + +### DUE-003: Explanation Placement + +Decision: richer explanations should use this placement model: + +- Short helper text below labels only when it prevents common mistakes. +- Tooltips for icon-only controls and compact terms. +- Expandable "Why?" or "Details" blocks for contextual explanations. +- Right-side detail panel or review step for preflight/provenance/diagnostics. + +Avoid permanently visible paragraphs inside dense admin cards. + +### DUE-004: Advanced Options Contract + +Decision: advanced options are fields that are needed for control, compatibility, +or diagnostics but are not part of the common setup path. + +- protocol-specific endpoints, ports, path overrides, TLS/signing toggles, + timeout/retry tuning, raw headers, migration/destructive flags, and fallback + compatibility settings are advanced. +- names, descriptions, provider type, base URL, ownership, policy mode, and + basic credentials are not advanced. + +Advanced fields must still be editable through typed controls. + +### DUE-005: Blocker Language Contract + +Decision: all disabled or blocked primary actions should use a shared structured +reason object. + +Shape: + +- `summary`: short plain-language reason. +- `details`: optional explanation. +- `required_action`: what needs to happen. +- `actor`: who can do it, for example system administrator or tenant admin. +- `target`: where to go or which setting/capability is missing. +- `technical_details`: optional expandable developer/operator data. + +For simple missing required fields, highlight the fields and put the structured +reason in a compact hover/focus bubble on the disabled primary action instead of +adding a large persistent warning block. + +### DUE-006: First Migration Surface + +Decision: convert file connectors first, then mail servers. They share the same +server/credential/policy model, are high-value, and will prove the reusable +patterns quickly. + +### DUE-007: Adaptive Create/Edit Forms + +Decision: ordinary create/edit dialogs should show the full editable state in an +adaptive form, not force a linear wizard. + +- The user chooses a type, provider, credential mode, or policy mode. +- The dialog immediately adjusts to show only the fields relevant to that state. +- Editing an existing object uses the same field grouping and layout as creating + it, so users can recognize the state they configured. +- Discovery and test actions must give visible feedback in the same dialog: + directly usable, discovered alternative, credentials needed/rejected, or not + found with the next action the user can take. +- Wizard shells remain available for assisted setup, first-run guidance, + imports, discovery-heavy flows, and operational preflight workflows. + +## Implementation Sequence + +| Phase | Scope | Output | +| --- | --- | --- | +| 0 | UX inventory | List every admin/settings/configuration surface, classify it, and record whether it violates a binding decision. | +| 1 | Core primitives | Shared wizard shell, advanced panel, help affordance, blocker callout, problem list, review step, and discovery/test result components. | +| 2 | File connectors | Adaptive create/edit for provider/server, credential, discovery/test, ownership/policy, plus optional assisted wizard later. | +| 3 | Mail servers | Same adaptive pattern as files, adapted to server/credential/policy and test-send/test-login behavior. | +| 4 | Policy/retention editors | Effective value first, provenance visible, override/edit in modal, blocked edits explained. | +| 5 | Module/package operations | Step-based install/uninstall flow with preflight, maintenance, daemon handoff, migration, and rollback explanation. | +| 6 | Remaining settings/admin screens | Apply inventory findings by priority and remove one-off layouts. | + +## Current Surface Inventory + +This is the phase-0 inventory baseline. It should be extended as each screen is +converted or reviewed. + +| Surface | Repository | UX State | Next Action | +| --- | --- | --- | --- | +| File connector settings | `govoplan-files` | First adaptive modal slice started: connections and credentials now use full-state create/edit forms with conditional fields, advanced panels, and blocker primitives. Wizard shell is retained for later assisted setup. Central policy card still needs a layered editor. | Finish provider discovery/test-in-flow, then convert policy editing. | +| Mail server settings | `govoplan-mail` / `govoplan-core` | Uses the shared server/credential model visually, but create/edit still needs the same adaptive pattern as files. | Migrate to adaptive server/credential/policy dialogs, with optional assisted wizard later. | +| Connector policy/effective rows | `govoplan-core`, module UIs | Effective-policy direction exists, but many editors still expose broad option sets. | Put effective value first, move overrides into modal, and explain blocked edits. | +| Admin module management | `govoplan-admin` | Has preflight concepts, but operational choices are still technical and dense. | Convert install/uninstall/package changes to operator wizards. | +| Configuration packages | `govoplan-admin` | Catalog/import work exists, but package editing can still drift toward technical fields. | Add guided import/review/problem-list flow. | +| Retention and privacy | `govoplan-core` | Functional editor exists; consequence language and provenance can be stronger. | Layer advanced retention options and add review for broad changes. | +| API keys | `govoplan-access` / admin UI | Security-sensitive creation needs least-privilege guidance. | Add scoped creation wizard with expiry/owner review. | +| User settings | `govoplan-core` | Preferences persistence exists; interface navigation issue was fixed earlier, but the surface still needs UX review. | Keep simple sections, remove double-click traps, and add quiet explanations. | + +## Impact Index + +| Surface | Current Risk | Expected Pattern | +| --- | --- | --- | +| File connectors | Too many technical fields and unclear setup order. | Adaptive create/edit form with optional assisted wizard, discovery/test, credential binding, policy review, and advanced protocol panel. | +| Mail servers | Similar server/credential/policy concepts risk diverging from files. | Same tree/list and adaptive server/credential/policy model as file connectors. | +| Connector credentials | Security-sensitive details can overwhelm users. | Separate credential flow with secret-reference language and test result explanation. | +| Policy and effective settings | Users need to know why a value is inherited or locked. | Effective row first, source/provenance, local override action, actionable blocked reason. | +| Module install/uninstall | Operationally risky, currently inherently technical. | Operator wizard with preflight, maintenance, daemon handoff, review, and rollback explanation. | +| Configuration packages | Could become package JSON editing. | Package catalog/import wizard using provider data requirements and problem lists. | +| Retention/privacy | High-risk settings need explanation and provenance. | Layered editor with plain-language consequences and review. | +| Automation/workflow commands | Hidden side effects would undermine accountability. | Action/effect preview, system-actor display, command record, retry/quarantine/manual states, and audit links. | +| Postbox and encrypted communication | Retraction and access can be misunderstood. | Honest key-fetch/decryption state, expiry limits, recipient/device access provenance, and delivery evidence. | +| API keys | Security-sensitive creation and scope selection. | Scoped creation wizard, least-privilege suggestions, clear expiry/owner explanation. | +| User settings | Needs clarity and persistence across profile/interface/preferences. | Simple settings sections with immediate feedback and no double-click navigation traps. | + +## Review Checklist + +Every new or changed admin/configuration surface should answer: + +- What is the common path, and is it visible without noise? +- Which fields are advanced, and are they collapsed by default? +- Is every configuration value editable through typed controls? +- Does the flow avoid JSON as the primary editor? +- Does the screen explain disabled actions and failed validation in plain + language? +- Does it say who can fix a blocker and where? +- Does it reuse existing core patterns for wizard steps, problem lists, modals, + help, and review? +- Is there a review or preflight step before broad, destructive, or risky + changes? +- Does the action surface show consequence, reversibility, and audit evidence + when rights, duties, records, money, communication, external systems, or + workflow state are affected? +- If automation is involved, can the user see the trigger, system actor, + observed effects, and failure/manual-intervention state? +- Are technical details available without being the first thing the user sees? + +## Revision Rule + +When a UX decision changes: + +1. Update this ledger. +2. Update the affected shared components. +3. Update existing screens listed in the impact index. +4. Add or update Gitea issues for any remaining surfaces that still follow the + old decision. +5. Sync the wiki. diff --git a/docs/audits/2026-07-09-dependency-audit.md b/docs/audits/2026-07-09-dependency-audit.md new file mode 100644 index 0000000..0df108a --- /dev/null +++ b/docs/audits/2026-07-09-dependency-audit.md @@ -0,0 +1,71 @@ +# Dependency Audit - 2026-07-09 + +Commands: + +```bash +cd /mnt/DATA/git/govoplan-core +bash scripts/check-dependency-audits.sh +``` + +Status: remediated. + +Initial result: + +- Python audit failed: 24 advisories were reported across `cryptography`, + `pip`, `python-multipart`, `pyzipper`, and `starlette`. +- npm production audit passed: `npm audit --omit=dev` reported 0 + vulnerabilities. + +Python findings: + +| Package | Installed | Advisory count | Minimum reported fix | +| --- | ---: | ---: | --- | +| `cryptography` | `44.0.0` | 5 | `48.0.1` | +| `pip` | `26.0.1` | 3 | `26.1.2` | +| `python-multipart` | `0.0.17` | 7 | `0.0.31` | +| `pyzipper` | `0.3.6` | 1 | `0.4.0` | +| `starlette` | `0.41.3` | 8 | `1.3.1` | + +Private GovOPlaN packages were skipped by `pip-audit` because they are not +published on PyPI. That is expected for local editable development installs. + +The remediation below upgrades those dependencies and records the compatibility +checks run against core, files, and campaign behavior. + +## Remediation + +Remediation applied on 2026-07-09: + +- upgraded core's FastAPI floor to `fastapi>=0.139,<1`, resolving Starlette to + `starlette==1.3.1` +- upgraded core's cryptography floor to `cryptography>=48.0.1,<50`, resolving + to `cryptography==49.0.0` +- declared the files module upload parser dependency as + `python-multipart>=0.0.31,<1`, resolving to `python-multipart==0.0.32` +- upgraded the campaign ZIP dependency to `pyzipper>=0.4,<1`, resolving to + `pyzipper==0.4.0` +- upgraded the local audit environment to `pip==26.1.2` +- removed the obsolete local `govoplan-module-multimailer` editable install + from the audit environment so the audit reflects the split module product + +Post-remediation result: + +- `bash scripts/check-dependency-audits.sh`: passed, no known Python + vulnerabilities found and npm production audit reported 0 vulnerabilities. +- `python -m pip check`: passed. +- `bash scripts/check-module-matrix.sh`: passed. +- `python -m unittest tests.test_api_smoke`: passed. +- campaign encrypted/plain ZIP smoke with `pyzipper==0.4.0`: passed. + +Notes: + +- `pip-audit` still reports private GovOPlaN packages as skipped because they + are not published on PyPI. That is expected for editable local development + installs. +- `httpx2>=2.5,<3` is included in development requirements so Starlette's + `TestClient` uses the non-deprecated backend. `httpx==0.28.1` remains in dev + requirements for tests that mock connector HTTP responses directly. +- Split-module API routers now use Starlette's renamed + `HTTP_422_UNPROCESSABLE_CONTENT` status constant. This preserves the 422 + status code while avoiding the deprecated `HTTP_422_UNPROCESSABLE_ENTITY` + alias. diff --git a/docs/gitea-labels.json b/docs/gitea-labels.json index fc4691b..74dceb9 100644 --- a/docs/gitea-labels.json +++ b/docs/gitea-labels.json @@ -179,6 +179,12 @@ "description": "GovOPlaN Identity Trust module behavior or integration.", "exclusive": false }, + { + "name": "module/identity", + "color": "bfd4f2", + "description": "GovOPlaN Identity module behavior or integration.", + "exclusive": false + }, { "name": "module/idm", "color": "0052cc", @@ -209,6 +215,12 @@ "description": "GovOPlaN Ops module behavior or integration.", "exclusive": false }, + { + "name": "module/organizations", + "color": "bfdadc", + "description": "GovOPlaN Organizations module behavior or integration.", + "exclusive": false + }, { "name": "module/payments", "color": "bfd4f2", @@ -227,6 +239,12 @@ "description": "GovOPlaN Portal module behavior or integration.", "exclusive": false }, + { + "name": "module/postbox", + "color": "d93f0b", + "description": "GovOPlaN Postbox module behavior or integration.", + "exclusive": false + }, { "name": "module/reporting", "color": "c2e0c6", diff --git a/docs/migration-release-baselines.json b/docs/migration-release-baselines.json new file mode 100644 index 0000000..953559c --- /dev/null +++ b/docs/migration-release-baselines.json @@ -0,0 +1,4 @@ +{ + "version": 1, + "releases": [] +} diff --git a/docs/module-package-catalog.example.json b/docs/module-package-catalog.example.json index 3d62630..888f7e2 100644 --- a/docs/module-package-catalog.example.json +++ b/docs/module-package-catalog.example.json @@ -15,6 +15,22 @@ "python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4", "webui_package": "@govoplan/files-webui", "webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4", + "artifact_integrity": { + "python": { + "ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4", + "sha256": "", + "sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-files-0.1.4.spdx.json", + "provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-files-0.1.4.intoto.jsonl", + "git_ref": "refs/tags/v0.1.4" + }, + "webui": { + "ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4", + "sha256": "", + "sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-files-webui-0.1.4.spdx.json", + "provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-files-webui-0.1.4.intoto.jsonl", + "git_ref": "refs/tags/v0.1.4" + } + }, "license_features": ["module.files"], "tags": ["official", "service-module"] }, @@ -28,8 +44,53 @@ "python_ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4", "webui_package": "@govoplan/mail-webui", "webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4", + "artifact_integrity": { + "python": { + "ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4", + "sha256": "", + "sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-mail-0.1.4.spdx.json", + "provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-mail-0.1.4.intoto.jsonl", + "git_ref": "refs/tags/v0.1.4" + }, + "webui": { + "ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4", + "sha256": "", + "sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-mail-webui-0.1.4.spdx.json", + "provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-mail-webui-0.1.4.intoto.jsonl", + "git_ref": "refs/tags/v0.1.4" + } + }, "license_features": ["module.mail"], "tags": ["official", "service-module"] + }, + { + "module_id": "dashboard", + "name": "Dashboard", + "description": "Configurable user home assembled from module-provided widgets.", + "version": "0.1.6", + "action": "install", + "python_package": "govoplan-dashboard", + "python_ref": "govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6", + "webui_package": "@govoplan/dashboard-webui", + "webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6", + "artifact_integrity": { + "python": { + "ref": "govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6", + "sha256": "", + "sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-dashboard-0.1.6.spdx.json", + "provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-dashboard-0.1.6.intoto.jsonl", + "git_ref": "refs/tags/v0.1.6" + }, + "webui": { + "ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6", + "sha256": "", + "sbom_url": "https://govoplan.add-ideas.de/sbom/govoplan-dashboard-webui-0.1.6.spdx.json", + "provenance_url": "https://govoplan.add-ideas.de/provenance/govoplan-dashboard-webui-0.1.6.intoto.jsonl", + "git_ref": "refs/tags/v0.1.6" + } + }, + "license_features": ["module.dashboard"], + "tags": ["official", "platform-module"] } ], "signatures": [ diff --git a/pyproject.toml b/pyproject.toml index 61ea371..3033ae0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,10 @@ license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ "SQLAlchemy>=2.0,<3", - "fastapi>=0.115,<1", + "fastapi>=0.139,<1", "pydantic>=2,<3", "pydantic-settings>=2,<3", - "cryptography>=44,<45", + "cryptography>=48.0.1,<50", "celery>=5,<6", "redis>=5,<6", "alembic>=1,<2", @@ -34,8 +34,10 @@ govoplan-module-installer = "govoplan_core.commands.module_installer:main" [project.optional-dependencies] server = [ + "psycopg[binary]>=3.2,<4", "uvicorn[standard]>=0.32,<1", ] dev = [ "httpx==0.28.1", + "httpx2>=2.5,<3", ] diff --git a/requirements-dev.txt b/requirements-dev.txt index 850259f..b5002b2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,11 +1,18 @@ -e .[server] -e ../govoplan-tenancy +-e ../govoplan-organizations +-e ../govoplan-identity -e ../govoplan-access -e ../govoplan-admin -e ../govoplan-policy -e ../govoplan-audit +-e ../govoplan-dashboard -e ../govoplan-files -e ../govoplan-mail -e ../govoplan-campaign -e ../govoplan-calendar +-e ../govoplan-docs +-e ../govoplan-ops httpx==0.28.1 +httpx2>=2.5,<3 +pip-audit>=2.9,<3 diff --git a/requirements-release.txt b/requirements-release.txt index ed09021..92009ed 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1,12 +1,17 @@ # Release install from tagged module repositories. # Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release. .[server] -govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.4 -govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.4 -govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.4 -govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.4 -govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.4 -govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4 -govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4 -govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.4 -govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.4 +govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.6 +govoplan-organizations @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-organizations.git@v0.1.6 +govoplan-identity @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-identity.git@v0.1.6 +govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.6 +govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.6 +govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.6 +govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.6 +govoplan-dashboard @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git@v0.1.6 +govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.6 +govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.6 +govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.6 +govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.6 +govoplan-docs @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git@v0.1.6 +govoplan-ops @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git@v0.1.6 diff --git a/scripts/check-dependency-audits.sh b/scripts/check-dependency-audits.sh new file mode 100644 index 0000000..53f2ab4 --- /dev/null +++ b/scripts/check-dependency-audits.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" +NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}" +NODE_BIN="$(dirname "$NPM")" + +if [[ ! -x "$PYTHON" ]]; then + PYTHON=python +fi +if [[ ! -x "$NPM" ]]; then + NPM=npm + NODE_BIN="$(dirname "$(command -v "$NPM")")" +fi + +cd "$ROOT" +CHECK_TESTCLIENT_DEPRECATIONS=auto bash "$ROOT/scripts/check-dependency-hygiene.sh" + +if ! "$PYTHON" -m pip_audit --version >/dev/null 2>&1; then + echo "pip-audit is not installed. Install dev requirements first:" >&2 + echo " $PYTHON -m pip install -r $ROOT/requirements-dev.txt" >&2 + exit 127 +fi + +python_status=0 +npm_status=0 + +"$PYTHON" -m pip_audit --progress-spinner off || python_status=$? + +cd "$ROOT/webui" +PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" audit --omit=dev || npm_status=$? + +if [[ "$python_status" -ne 0 || "$npm_status" -ne 0 ]]; then + echo "Dependency audit failed: python=$python_status npm=$npm_status" >&2 + exit 1 +fi diff --git a/scripts/check-dependency-hygiene.sh b/scripts/check-dependency-hygiene.sh new file mode 100644 index 0000000..4956e3f --- /dev/null +++ b/scripts/check-dependency-hygiene.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" +CHECK_TESTCLIENT_DEPRECATIONS="${CHECK_TESTCLIENT_DEPRECATIONS:-auto}" + +if [[ ! -x "$PYTHON" ]]; then + PYTHON=python +fi + +cd "$ROOT" + +"$PYTHON" - <<'PY' +import importlib.util +import sys + +if importlib.util.find_spec("pip") is None: + print("Dependency hygiene failed: this Python environment cannot import pip.", file=sys.stderr) + print("Recreate or repair the venv, then reinstall requirements:", file=sys.stderr) + print(" python -m venv .venv", file=sys.stderr) + print(" ./.venv/bin/python -m pip install --upgrade pip", file=sys.stderr) + print(" ./.venv/bin/python -m pip install -r requirements-dev.txt", file=sys.stderr) + raise SystemExit(127) +PY + +"$PYTHON" -m pip check + +"$PYTHON" - <<'PY' +from __future__ import annotations + +import importlib.metadata +import pathlib +import sys + +prefix = pathlib.Path(sys.prefix) +legacy_patterns = ( + "__editable__.govoplan_module_multimailer-*.pth", + "__editable___govoplan_module_multimailer_*_finder.py", + "govoplan_module_multimailer-*.dist-info", +) +problems: list[pathlib.Path] = [] + +for site_packages in (prefix / "lib").glob("python*/site-packages"): + for pattern in legacy_patterns: + problems.extend(site_packages.glob(pattern)) + +for dist in importlib.metadata.distributions(): + if dist.metadata.get("Name", "").lower() == "govoplan-module-multimailer": + problems.append(pathlib.Path(str(dist.locate_file("")))) + +if problems: + print("Dependency hygiene failed: stale legacy multimailer install metadata found.", file=sys.stderr) + for path in sorted(set(problems)): + print(f" {path}", file=sys.stderr) + print("Remove the stale files or recreate the venv. This package pins obsolete dependencies.", file=sys.stderr) + raise SystemExit(1) + +print("No stale legacy multimailer package metadata found.") +PY + +"$PYTHON" - <<'PY' +from __future__ import annotations + +import pathlib +import sys + +root = pathlib.Path.cwd() +scan_roots = [ + root / "src", + root / "tests", + root.parent / "govoplan-access" / "src", + root.parent / "govoplan-admin" / "src", + root.parent / "govoplan-audit" / "src", + root.parent / "govoplan-calendar" / "src", + root.parent / "govoplan-campaign" / "src", + root.parent / "govoplan-dashboard" / "src", + root.parent / "govoplan-files" / "src", + root.parent / "govoplan-identity" / "src", + root.parent / "govoplan-mail" / "src", + root.parent / "govoplan-ops" / "src", + root.parent / "govoplan-organizations" / "src", + root.parent / "govoplan-policy" / "src", + root.parent / "govoplan-tenancy" / "src", +] +deprecated_tokens = { + "HTTP_422_UNPROCESSABLE_ENTITY": "Use HTTP_422_UNPROCESSABLE_CONTENT; the numeric status remains 422.", +} + +hits: list[str] = [] +for scan_root in scan_roots: + if not scan_root.exists(): + continue + for path in scan_root.rglob("*.py"): + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + for token, replacement in deprecated_tokens.items(): + if token in text: + hits.append(f"{path}: deprecated {token}. {replacement}") + +if hits: + print("Dependency hygiene failed: deprecated framework constants are still used.", file=sys.stderr) + print("\n".join(hits), file=sys.stderr) + raise SystemExit(1) + +print("Deprecated framework constant scan passed.") +PY + +RUN_TESTCLIENT_DEPRECATION_SMOKE=0 +case "$CHECK_TESTCLIENT_DEPRECATIONS" in + 0|false|False|no|No) + echo "TestClient deprecation smoke skipped by CHECK_TESTCLIENT_DEPRECATIONS=$CHECK_TESTCLIENT_DEPRECATIONS" + ;; + auto) + if "$PYTHON" - <<'PY' +import importlib.util +raise SystemExit(0 if importlib.util.find_spec("fastapi") and importlib.util.find_spec("starlette") and importlib.util.find_spec("httpx2") else 1) +PY + then + RUN_TESTCLIENT_DEPRECATION_SMOKE=1 + else + echo "TestClient deprecation smoke skipped; install dev requirements to enable it." + fi + ;; + *) + RUN_TESTCLIENT_DEPRECATION_SMOKE=1 + ;; +esac + +if [[ "$RUN_TESTCLIENT_DEPRECATION_SMOKE" -eq 1 ]]; then + "$PYTHON" - <<'PY' +import warnings + +from starlette.exceptions import StarletteDeprecationWarning + +warnings.simplefilter("error", StarletteDeprecationWarning) + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +app = FastAPI() + +@app.get("/dependency-hygiene-ping") +def ping() -> dict[str, bool]: + return {"ok": True} + +with TestClient(app) as client: + response = client.get("/dependency-hygiene-ping") + assert response.status_code == 200 + assert response.json() == {"ok": True} + +print("TestClient deprecation smoke passed.") +PY +fi + +echo "Dependency hygiene check passed." diff --git a/scripts/check-focused.sh b/scripts/check-focused.sh index f25afd8..4c3cad8 100644 --- a/scripts/check-focused.sh +++ b/scripts/check-focused.sh @@ -5,11 +5,19 @@ ROOT="/mnt/DATA/git/govoplan-core" NODE="/home/zemion/.nvm/versions/node/v22.22.3/bin" NPM="$NODE/npm" WEBUI_BIN="$ROOT/webui/node_modules/.bin" +NPM_USERCONFIG="$(mktemp "${TMPDIR:-/tmp}/govoplan-npmrc.XXXXXXXX")" + +trap 'rm -f "$NPM_USERCONFIG"' EXIT export PATH="$WEBUI_BIN:$NODE:$PATH" +export NPM_CONFIG_USERCONFIG="$NPM_USERCONFIG" +export GOVOPLAN_NPM_USERCONFIG="$NPM_USERCONFIG" +unset npm_config_tmp NPM_CONFIG_TMP cd "$ROOT" +CHECK_TESTCLIENT_DEPRECATIONS=1 bash scripts/check-dependency-hygiene.sh + ./.venv/bin/python - <<'PY' import ast import pathlib @@ -20,6 +28,8 @@ roots = [ pathlib.Path("/mnt/DATA/git/govoplan-access/src"), pathlib.Path("/mnt/DATA/git/govoplan-admin/src"), pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"), + pathlib.Path("/mnt/DATA/git/govoplan-organizations/src"), + pathlib.Path("/mnt/DATA/git/govoplan-identity/src"), pathlib.Path("/mnt/DATA/git/govoplan-policy/src"), pathlib.Path("/mnt/DATA/git/govoplan-audit/src"), pathlib.Path("/mnt/DATA/git/govoplan-mail/src"), @@ -47,7 +57,7 @@ if errors: print(f"AST syntax check passed for {count} Python files") PY -./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")' +./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_access.backend.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")' ./.venv/bin/python scripts/check_dependency_boundaries.py ./.venv/bin/python -m unittest tests.test_module_system ./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests diff --git a/scripts/check-module-matrix.sh b/scripts/check-module-matrix.sh new file mode 100644 index 0000000..200ade9 --- /dev/null +++ b/scripts/check-module-matrix.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" +NPM="${NPM:-/home/zemion/.nvm/versions/node/v22.22.3/bin/npm}" +NODE_BIN="$(dirname "$NPM")" + +if [[ ! -x "$PYTHON" ]]; then + PYTHON=python +fi +if [[ ! -x "$NPM" ]]; then + NPM=npm + NODE_BIN="$(dirname "$(command -v "$NPM")")" +fi + +cd "$ROOT" +"$PYTHON" -m unittest tests.test_module_system tests.test_access_contracts +"$PYTHON" scripts/check_dependency_boundaries.py + +cd "$ROOT/webui" +PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-capabilities +PATH="$ROOT/webui/node_modules/.bin:$NODE_BIN:$PATH" "$NPM" run test:module-permutations diff --git a/scripts/check_dependency_boundaries.py b/scripts/check_dependency_boundaries.py index ef71dd9..7dbc013 100644 --- a/scripts/check_dependency_boundaries.py +++ b/scripts/check_dependency_boundaries.py @@ -11,8 +11,11 @@ REPOS = { "access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access", "admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin", "tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy", + "organizations": ROOT.parent / "govoplan-organizations" / "src" / "govoplan_organizations", + "identity": ROOT.parent / "govoplan-identity" / "src" / "govoplan_identity", "policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy", "audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit", + "dashboard": ROOT.parent / "govoplan-dashboard" / "src" / "govoplan_dashboard", "files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files", "mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail", "campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign", @@ -22,15 +25,18 @@ PREFIXES = { "access": "govoplan_access", "admin": "govoplan_admin", "tenancy": "govoplan_tenancy", + "organizations": "govoplan_organizations", + "identity": "govoplan_identity", "policy": "govoplan_policy", "audit": "govoplan_audit", + "dashboard": "govoplan_dashboard", "files": "govoplan_files", "mail": "govoplan_mail", "campaign": "govoplan_campaign", } FEATURE_OWNERS = ("files", "mail", "campaign") -PLATFORM_OWNERS = ("admin", "tenancy", "policy", "audit") -PUBLIC_ACCESS_IMPORTS = ("govoplan_access.backend.auth.dependencies",) +PLATFORM_OWNERS = ("admin", "tenancy", "organizations", "identity", "policy", "audit", "dashboard") +PUBLIC_ACCESS_IMPORTS = ("govoplan_access.auth",) @dataclass(frozen=True) diff --git a/scripts/generate-release-catalog.py b/scripts/generate-release-catalog.py index 5c8cba3..4d3ac0e 100644 --- a/scripts/generate-release-catalog.py +++ b/scripts/generate-release-catalog.py @@ -38,6 +38,22 @@ CATALOG_MODULES = ( description="Tenant registry, tenant settings, and tenant resolution platform module.", tags=("official", "platform-module"), ), + CatalogModule( + module_id="organizations", + repo="govoplan-organizations", + python_package="govoplan-organizations", + name="Organizations", + description="Organization units, functions, and account-held function assignments.", + tags=("official", "platform-module"), + ), + CatalogModule( + module_id="identity", + repo="govoplan-identity", + python_package="govoplan-identity", + name="Identity", + description="Canonical identities and links between identities and platform accounts.", + tags=("official", "platform-module"), + ), CatalogModule( module_id="access", repo="govoplan-access", @@ -72,6 +88,15 @@ CATALOG_MODULES = ( description="Audit-log storage and audit administration routes.", tags=("official", "platform-module"), ), + CatalogModule( + module_id="dashboard", + repo="govoplan-dashboard", + python_package="govoplan-dashboard", + name="Dashboard", + description="Configurable user home assembled from module-provided dashboard widgets.", + tags=("official", "platform-module"), + webui_package="@govoplan/dashboard-webui", + ), CatalogModule( module_id="files", repo="govoplan-files", @@ -108,6 +133,24 @@ CATALOG_MODULES = ( tags=("official", "service-module"), webui_package="@govoplan/calendar-webui", ), + CatalogModule( + module_id="docs", + repo="govoplan-docs", + python_package="govoplan-docs", + name="Docs", + description="Configured-system documentation and evidence-aware help surfaces.", + tags=("official", "platform-module"), + webui_package="@govoplan/docs-webui", + ), + CatalogModule( + module_id="ops", + repo="govoplan-ops", + python_package="govoplan-ops", + name="Ops", + description="Runtime health, deployment profile, worker split, and sizing visibility.", + tags=("official", "platform-module"), + webui_package="@govoplan/ops-webui", + ), ) diff --git a/scripts/gitea-backlog-import.py b/scripts/gitea-backlog-import.py index 4e122ab..9950768 100644 --- a/scripts/gitea-backlog-import.py +++ b/scripts/gitea-backlog-import.py @@ -12,11 +12,10 @@ import re import sys from typing import Any, Iterable -from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token +from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md") -DEFAULT_ACCESS_PLAN = pathlib.Path("/mnt/DATA/git/govoplan-core/docs/ACCESS_EXTRACTION_PLAN.md") DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md") REPO_ROOTS = { @@ -62,7 +61,7 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values") parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG) - parser.add_argument("--access-plan", type=pathlib.Path, default=DEFAULT_ACCESS_PLAN) + parser.add_argument("--access-plan", type=pathlib.Path, help="optional legacy access extraction plan to import") parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO) parser.add_argument("--apply", action="store_true", help="create missing Gitea issues") args = parser.parse_args() @@ -128,7 +127,7 @@ def main() -> int: def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]: if args.product_backlog.exists(): yield from parse_product_backlog(args.product_backlog) - if args.access_plan.exists(): + if args.access_plan and args.access_plan.exists(): yield from parse_access_plan(args.access_plan) if args.web_todo.exists(): yield from parse_simple_todo(args.web_todo) @@ -539,8 +538,7 @@ def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]: root = repo_root(REPO_ROOTS[repo_key]) target = infer_target(root) client = GiteaClient(target, token) - labels_payload = client.paginate(repo_path(target.owner, target.repo, "/labels")) - label_ids = {str(label["name"]): int(label["id"]) for label in labels_payload if "name" in label and "id" in label} + label_ids = label_ids_by_name(client, target.owner, target.repo) issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100) fingerprints: set[str] = set() titles: set[str] = set() diff --git a/scripts/gitea-import-all-backlogs.py b/scripts/gitea-import-all-backlogs.py index 0d5f370..c9a2f43 100644 --- a/scripts/gitea-import-all-backlogs.py +++ b/scripts/gitea-import-all-backlogs.py @@ -14,7 +14,7 @@ import subprocess import sys from typing import Any, Iterable -from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token +from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, require_token GIT_ROOT = pathlib.Path("/mnt/DATA/git") @@ -246,9 +246,7 @@ def is_excluded_repo_file(path: pathlib.Path) -> bool: return True if text.endswith("/docs/GITEA_ISSUES.md"): return True - if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"): - return True - if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"): + if text.endswith("/docs/GOVOPLAN_MASTER_ROADMAP.md"): return True if text.endswith("/govoplan-web/TODO.md"): return True @@ -537,8 +535,7 @@ def grouped_counts(candidates: list[Candidate]) -> dict[str, int]: def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState: target = infer_target(repo.root) client = GiteaClient(target, token) - labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50) - label_ids = {str(label["name"]): int(label["id"]) for label in labels} + label_ids = label_ids_by_name(client, target.owner, target.repo) if apply: for name, color, description in GENERIC_LABELS: if name in label_ids: diff --git a/scripts/gitea-migrate-org-labels.py b/scripts/gitea-migrate-org-labels.py new file mode 100644 index 0000000..6a2f137 --- /dev/null +++ b/scripts/gitea-migrate-org-labels.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Move issue labels from repository-local labels to organization labels.""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import re +import sys +from typing import Any + +from gitea_common import ( + GiteaClient, + GiteaError, + infer_target, + load_dotenv, + org_path, + repo_path, + repo_root, + require_token, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root used for Gitea URL inference") + parser.add_argument("--remote", default="origin", help="git remote to use for target inference") + parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env")) + parser.add_argument("--org", help="organization owner; defaults to inferred owner") + parser.add_argument("--repo", action="append", default=[], help="repository name to process; may be repeated") + parser.add_argument("--repo-regex", default=".*", help="only process repositories whose name matches this regex") + parser.add_argument( + "--issue-mode", + choices=("replace", "delete-add"), + default="replace", + help="replace labels in one request, or delete local ids before adding org ids", + ) + parser.add_argument("--delete-repo-labels", action="store_true", help="delete matching repository-local labels after issue and PR migration") + parser.add_argument("--apply", action="store_true", help="apply changes; omit for dry-run") + args = parser.parse_args() + + try: + root = repo_root(args.root) + load_dotenv(args.env_file or root / ".env") + target = infer_target(root, args.remote) + owner = args.org or target.owner + client = GiteaClient(target, require_token() if args.apply else os.environ.get("GITEA_TOKEN")) + if client.token is None: + raise GiteaError("GITEA_TOKEN is required for dry-run and apply because migration inspects live issue labels.") + + repo_filter = re.compile(args.repo_regex) + org_labels = _labels_by_name(client.paginate(org_path(owner, "/labels"), limit=100)) + if not org_labels: + raise GiteaError(f"{owner} has no organization labels") + + repos = _selected_repositories(client, owner, args.repo, repo_filter) + totals = Totals() + print(f"Organization: {owner}") + print(f"Repositories: {len(repos)}") + print(f"Mode: {'apply' if args.apply else 'dry-run'}") + print(f"Delete repository labels: {args.delete_repo_labels}") + + for index, repo in enumerate(repos, start=1): + repo_result = migrate_repository( + client, + owner=owner, + repo=repo, + org_labels=org_labels, + issue_mode=args.issue_mode, + apply=args.apply, + delete_repo_labels=args.delete_repo_labels, + ) + totals.add(repo_result) + if repo_result.has_work: + print( + f"[{index}/{len(repos)}] {repo}: " + f"issue-labels={repo_result.issue_label_migrations}, " + f"delete-labels={repo_result.repo_label_deletions}, " + f"errors={len(repo_result.errors)}" + ) + for error in repo_result.errors: + print(f" error: {error}") + else: + print(f"[{index}/{len(repos)}] {repo}: no matching local labels") + + print("Summary:") + print(f" repositories processed: {totals.repositories}") + print(f" repositories with matching local labels: {totals.repositories_with_work}") + print(f" issue/PR local labels migrated: {totals.issue_label_migrations}") + print(f" repository labels deleted: {totals.repo_label_deletions}") + print(f" errors: {totals.errors}") + if totals.errors: + return 1 + return 0 + except GiteaError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +class RepoResult: + def __init__(self) -> None: + self.issue_label_migrations = 0 + self.repo_label_deletions = 0 + self.errors: list[str] = [] + self.matched_local_labels = 0 + + @property + def has_work(self) -> bool: + return bool(self.matched_local_labels or self.issue_label_migrations or self.repo_label_deletions or self.errors) + + +class Totals: + def __init__(self) -> None: + self.repositories = 0 + self.repositories_with_work = 0 + self.issue_label_migrations = 0 + self.repo_label_deletions = 0 + self.errors = 0 + + def add(self, result: RepoResult) -> None: + self.repositories += 1 + if result.has_work: + self.repositories_with_work += 1 + self.issue_label_migrations += result.issue_label_migrations + self.repo_label_deletions += result.repo_label_deletions + self.errors += len(result.errors) + + +def migrate_repository( + client: GiteaClient, + *, + owner: str, + repo: str, + org_labels: dict[str, dict[str, Any]], + issue_mode: str, + apply: bool, + delete_repo_labels: bool, +) -> RepoResult: + result = RepoResult() + repo_labels = client.paginate(repo_path(owner, repo, "/labels"), limit=100) + local_labels = { + str(label.get("name") or ""): label + for label in repo_labels + if str(label.get("name") or "") in org_labels + } + result.matched_local_labels = len(local_labels) + if not local_labels: + return result + + local_by_id = {_label_id(label): label for label in local_labels.values() if _label_id(label) is not None} + org_by_name = {name: _label_id(label) for name, label in org_labels.items()} + + for issue_type in ("issues", "pulls"): + issues = client.paginate( + repo_path(owner, repo, "/issues"), + query={"state": "all", "type": issue_type}, + limit=100, + ) + for issue in issues: + issue_number = int(issue.get("number") or issue.get("index")) + issue_label_ids = [_label_id(label) for label in issue.get("labels") or []] + issue_label_ids = [label_id for label_id in issue_label_ids if label_id is not None] + final_label_ids = list(issue_label_ids) + changed = False + migrated_count = 0 + local_ids_to_remove: list[int] = [] + org_ids_to_add: list[int] = [] + for label in issue.get("labels") or []: + local_id = _label_id(label) + if local_id is None or local_id not in local_by_id: + continue + name = str(label.get("name") or "") + org_id = org_by_name.get(name) + if org_id is None: + continue + local_ids_to_remove.append(local_id) + if org_id not in issue_label_ids and org_id not in org_ids_to_add: + org_ids_to_add.append(org_id) + final_label_ids = [label_id for label_id in final_label_ids if label_id != local_id] + if org_id not in final_label_ids: + final_label_ids.append(org_id) + changed = True + migrated_count += 1 + if changed: + if apply: + if issue_mode == "delete-add": + for local_id in local_ids_to_remove: + _remove_issue_label(client, owner, repo, issue_number, local_id) + if org_ids_to_add: + client.request_json( + "POST", + repo_path(owner, repo, f"/issues/{issue_number}/labels"), + body={"labels": org_ids_to_add}, + ) + else: + client.request_json( + "PUT", + repo_path(owner, repo, f"/issues/{issue_number}/labels"), + body={"labels": final_label_ids}, + ) + result.issue_label_migrations += migrated_count + + if delete_repo_labels: + for name, label in sorted(local_labels.items()): + label_id = _label_id(label) + if label_id is None: + result.errors.append(f"{name} has no label id") + continue + if apply: + try: + client.request_json("DELETE", repo_path(owner, repo, f"/labels/{label_id}")) + except GiteaError as exc: + result.errors.append(f"delete repository label {name}: {exc}") + continue + result.repo_label_deletions += 1 + + return result + + +def _selected_repositories(client: GiteaClient, owner: str, explicit: list[str], repo_filter: re.Pattern[str]) -> list[str]: + if explicit: + return sorted(set(explicit)) + repos = client.paginate(org_path(owner, "/repos"), query={"type": "all"}, limit=100) + names = sorted(str(repo.get("name") or "") for repo in repos if repo.get("name")) + return [name for name in names if repo_filter.search(name)] + + +def _labels_by_name(labels: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return {str(label.get("name")): label for label in labels if label.get("name") and _label_id(label) is not None} + + +def _label_id(label: dict[str, Any]) -> int | None: + value = label.get("id") or label.get("index") + if value is None: + return None + return int(value) + + +def _remove_issue_label(client: GiteaClient, owner: str, repo: str, issue_number: int, label_id: int) -> None: + try: + client.request_json("DELETE", repo_path(owner, repo, f"/issues/{issue_number}/labels/{label_id}")) + except GiteaError as exc: + if "HTTP 404" in str(exc): + return + raise + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-sync-labels.py b/scripts/gitea-sync-labels.py index bf9aa8d..b0fb644 100644 --- a/scripts/gitea-sync-labels.py +++ b/scripts/gitea-sync-labels.py @@ -15,6 +15,7 @@ from gitea_common import ( infer_target, load_dotenv, load_json, + org_path, repo_path, repo_root, require_token, @@ -30,6 +31,13 @@ def main() -> int: parser.add_argument("--remote", default="origin", help="git remote to use for target inference") parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE) parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values") + parser.add_argument( + "--scope", + choices=("repository", "organization"), + default="repository", + help="sync repository labels or organization labels", + ) + parser.add_argument("--org", help="organization name for --scope organization; defaults to inferred owner") parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea") args = parser.parse_args() @@ -39,8 +47,12 @@ def main() -> int: target = infer_target(root, args.remote) labels = _load_labels(args.labels_file) token = require_token() if args.apply else os.environ.get("GITEA_TOKEN") + org_name = args.org or target.owner - print(f"Target: {target.display}") + if args.scope == "organization": + print(f"Target: {target.base_url.rstrip('/')}/{org_name} organization labels") + else: + print(f"Target: {target.display}") print(f"Labels file: {args.labels_file}") if not args.apply and not token: @@ -50,7 +62,10 @@ def main() -> int: return 0 client = GiteaClient(target, token) - existing = _labels_by_name(client, target.owner, target.repo) + if args.scope == "organization": + existing = _org_labels_by_name(client, org_name) + else: + existing = _repo_labels_by_name(client, target.owner, target.repo) creates: list[dict[str, Any]] = [] updates: list[tuple[dict[str, Any], dict[str, Any]]] = [] @@ -71,18 +86,17 @@ def main() -> int: for label in creates: print(f"{'create' if args.apply else 'would create'} {label['name']}") if args.apply: - client.request_json( - "POST", - repo_path(target.owner, target.repo, "/labels"), - body=label, - ) + client.request_json("POST", _create_path(args.scope, org_name, target.owner, target.repo), body=label) for current, patch in updates: print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}") if args.apply: + label_id = current.get("id") or current.get("index") + if label_id is None: + raise GiteaError(f"{current['name']} has no label id in API response") client.request_json( "PATCH", - repo_path(target.owner, target.repo, f"/labels/{current['id']}"), + _update_path(args.scope, org_name, target.owner, target.repo, int(label_id)), body=patch, ) @@ -119,11 +133,28 @@ def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]: return labels -def _labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]: +def _repo_labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]: labels = client.paginate(repo_path(owner, repo, "/labels")) return {str(label.get("name")): label for label in labels} +def _org_labels_by_name(client: GiteaClient, owner: str) -> dict[str, dict[str, Any]]: + labels = client.paginate(org_path(owner, "/labels")) + return {str(label.get("name")): label for label in labels} + + +def _create_path(scope: str, org: str, owner: str, repo: str) -> str: + if scope == "organization": + return org_path(org, "/labels") + return repo_path(owner, repo, "/labels") + + +def _update_path(scope: str, org: str, owner: str, repo: str, label_id: int) -> str: + if scope == "organization": + return org_path(org, f"/labels/{label_id}") + return repo_path(owner, repo, f"/labels/{label_id}") + + def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]: patch: dict[str, Any] = {} if _normalize_color(current.get("color")) != desired["color"]: diff --git a/scripts/gitea-sync-wiki.py b/scripts/gitea-sync-wiki.py index 0b26731..de8fc12 100644 --- a/scripts/gitea-sync-wiki.py +++ b/scripts/gitea-sync-wiki.py @@ -73,6 +73,7 @@ def main() -> int: parser.add_argument("--transport", choices=("git", "api"), default="git", help="sync through the wiki git repository or the Gitea REST API") parser.add_argument("--wiki-cache-dir", type=pathlib.Path, default=pathlib.Path("/tmp/codex-gitea-wiki-sync"), help="local cache for wiki git checkouts") parser.add_argument("--commit-message", default="Sync wiki from project files", help="commit message used by git transport") + parser.add_argument("--prune-managed", action="store_true", help="delete managed wiki pages whose source files are no longer discovered") parser.add_argument("--apply", action="store_true") args = parser.parse_args() @@ -116,9 +117,17 @@ def main() -> int: overwrite_unmanaged=args.overwrite_unmanaged, commit_message=args.commit_message, write_index=not bool(args.page), + prune_managed=args.prune_managed and not bool(args.page), ) else: - sync_repo_wiki(repo, repo_sources, token, overwrite_unmanaged=args.overwrite_unmanaged, write_index=not bool(args.page)) + sync_repo_wiki( + repo, + repo_sources, + token, + overwrite_unmanaged=args.overwrite_unmanaged, + write_index=not bool(args.page), + prune_managed=args.prune_managed and not bool(args.page), + ) except GiteaError as exc: failures += 1 print(f"error syncing wiki for {repo_key}: {exc}", file=sys.stderr) @@ -267,7 +276,15 @@ def preview_sources(sources: list[WikiSource]) -> None: print(f" ... {len(repo_sources) - 80} more") -def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, overwrite_unmanaged: bool, write_index: bool = True) -> None: +def sync_repo_wiki( + repo: RepoInfo, + sources: list[WikiSource], + token: str, + *, + overwrite_unmanaged: bool, + write_index: bool = True, + prune_managed: bool = False, +) -> None: target = infer_target(repo.root) client = GiteaClient(target, token) existing_pages = list_existing_wiki_pages(client, target.owner, target.repo) @@ -311,6 +328,8 @@ def sync_repo_wiki(repo: RepoInfo, sources: list[WikiSource], token: str, *, ove ) else: print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync") + if prune_managed and write_index: + prune_managed_wiki_pages_api(client, target.owner, target.repo, existing_page_names, sources) def sync_repo_wiki_git( @@ -321,6 +340,7 @@ def sync_repo_wiki_git( overwrite_unmanaged: bool, commit_message: str, write_index: bool = True, + prune_managed: bool = False, ) -> None: target = infer_target(repo.root) wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir) @@ -353,6 +373,8 @@ def sync_repo_wiki_git( ) else: print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync") + if prune_managed and write_index: + prune_managed_wiki_pages_git(wiki_root, sources) if not git_has_changes(wiki_root): print(f"unchanged {target.owner}/{target.repo} wiki repository") return @@ -426,6 +448,38 @@ def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, o return "updated" +def prune_managed_wiki_pages_git(wiki_root: pathlib.Path, sources: list[WikiSource]) -> None: + desired = {f"{wiki_file_stem(source.page_title)}.md" for source in sources} + desired.add(f"{wiki_file_stem('Codex-Project-Index')}.md") + for path in sorted(wiki_root.glob("*.md")): + if path.name in desired: + continue + current = read_text(path) + if MANAGED_MARKER not in current: + continue + path.unlink() + print(f"deleted stale managed wiki page {path.name}") + + +def prune_managed_wiki_pages_api( + client: GiteaClient, + owner: str, + repo: str, + existing_page_names: dict[str, str], + sources: list[WikiSource], +) -> None: + desired = {source.page_title for source in sources} + desired.add("Codex-Project-Index") + for title, page_name in sorted(existing_page_names.items()): + if title in desired: + continue + current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}")) + if MANAGED_MARKER not in decode_wiki_content(current): + continue + client.request_json("DELETE", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}")) + print(f"deleted stale managed wiki page {owner}/{repo}:{title}") + + def wiki_file_stem(title: str) -> str: return re.sub(r"[/\\]+", "-", title).strip() or "Home" diff --git a/scripts/gitea-todo-import.py b/scripts/gitea-todo-import.py index 1354268..666f9ee 100644 --- a/scripts/gitea-todo-import.py +++ b/scripts/gitea-todo-import.py @@ -13,7 +13,7 @@ import subprocess import sys from typing import Any -from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token +from gitea_common import GiteaClient, GiteaError, infer_target, label_ids_by_name, load_dotenv, repo_path, repo_root, require_token MARKER_RE = re.compile( @@ -347,8 +347,7 @@ def truncate_title(title: str, limit: int = 180) -> str: def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]: - labels = client.paginate(repo_path(owner, repo, "/labels")) - return {str(label["name"]): int(label["id"]) for label in labels if "name" in label and "id" in label} + return label_ids_by_name(client, owner, repo) def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]: diff --git a/scripts/gitea_common.py b/scripts/gitea_common.py index 09279f4..c6211b6 100644 --- a/scripts/gitea_common.py +++ b/scripts/gitea_common.py @@ -197,6 +197,47 @@ def repo_path(owner: str, repo: str, suffix: str) -> str: return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}" +def org_path(owner: str, suffix: str) -> str: + return f"/orgs/{quote_path(owner)}{suffix}" + + +def labels_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, dict[str, Any]]: + """Return org labels plus optional repository labels, keyed by label name. + + Gitea stores organization labels separately from repository labels. Issue + APIs can use label ids from both scopes on supported instances, so issue + creation helpers should resolve both. Repository labels intentionally win + when a repo carries a local label with the same name. + """ + + labels: dict[str, dict[str, Any]] = {} + try: + for label in client.paginate(org_path(owner, "/labels")): + name = str(label.get("name") or "") + if name: + labels[name] = label + except GiteaError: + # User-owned repositories or older instances may not expose org labels. + pass + + if repo: + for label in client.paginate(repo_path(owner, repo, "/labels")): + name = str(label.get("name") or "") + if name: + labels[name] = label + return labels + + +def label_ids_by_name(client: GiteaClient, owner: str, repo: str | None = None) -> dict[str, int]: + ids: dict[str, int] = {} + for name, label in labels_by_name(client, owner, repo).items(): + label_id = label.get("id") or label.get("index") + if label_id is None: + continue + ids[name] = int(label_id) + return ids + + def _git_remote(root: pathlib.Path, remote_name: str) -> str: result = subprocess.run( ["git", "-C", str(root), "remote", "get-url", remote_name], diff --git a/scripts/launch-dev.sh b/scripts/launch-dev.sh index e7a0577..a4b7bc1 100644 --- a/scripts/launch-dev.sh +++ b/scripts/launch-dev.sh @@ -16,6 +16,7 @@ FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}" FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}" FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" +DEV_DATABASE_BACKEND="${GOVOPLAN_DEV_DATABASE_BACKEND:-postgres}" LOG_DIR="$ROOT/runtime/dev-launcher" BACKEND_LOG="$LOG_DIR/backend.log" @@ -31,6 +32,20 @@ fail() { exit 1 } +case "$DEV_DATABASE_BACKEND" in + postgres) + export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev}" + export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev}" + ;; + sqlite) + export DATABASE_URL="${DATABASE_URL:-sqlite:///$ROOT/runtime/multimailer-dev.db}" + ;; + *) + fail "Unsupported GOVOPLAN_DEV_DATABASE_BACKEND=$DEV_DATABASE_BACKEND. Use postgres or sqlite." + ;; +esac +export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}" + port_is_free() { "$PYTHON" - "$1" "$2" <<'PY' import socket @@ -139,6 +154,7 @@ GovOPlaN is running. Web UI: $FRONTEND_URL API: $BACKEND_URL/api/v1 Health: $BACKEND_URL/health +DB: $DATABASE_URL Logs: Backend: $BACKEND_LOG diff --git a/scripts/launch-production-like-dev.sh b/scripts/launch-production-like-dev.sh new file mode 100644 index 0000000..216c7fa --- /dev/null +++ b/scripts/launch-production-like-dev.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${GOVOPLAN_CORE_ROOT:-/mnt/DATA/git/govoplan-core}" +PROFILE_ROOT="$ROOT/dev/production-like" +PROFILE_ENV="${GOVOPLAN_PRODUCTION_LIKE_ENV:-$PROFILE_ROOT/.env}" +if [ ! -f "$PROFILE_ENV" ]; then + PROFILE_ENV="$PROFILE_ROOT/.env.example" +fi + +WEBUI_ROOT="$ROOT/webui" +PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" +NODE_BIN="${NODE_BIN:-/home/zemion/.nvm/versions/node/v22.22.3/bin}" +NPM="${NPM:-$NODE_BIN/npm}" +DOCKER="${DOCKER:-docker}" + +BACKEND_HOST="${GOVOPLAN_BACKEND_HOST:-127.0.0.1}" +BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}" +FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}" +FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}" +OPEN_BROWSER="${OPEN_BROWSER:-1}" +STOP_DEPENDENCIES_ON_EXIT="${GOVOPLAN_STOP_PROFILE_DEPENDENCIES_ON_EXIT:-0}" + +LOG_DIR="$ROOT/runtime/production-like/logs" +BACKEND_LOG="$LOG_DIR/backend.log" +WORKER_LOG="$LOG_DIR/worker.log" +FRONTEND_LOG="$LOG_DIR/frontend.log" +BACKEND_URL="http://$BACKEND_HOST:$BACKEND_PORT" +FRONTEND_URL="http://$FRONTEND_HOST:$FRONTEND_PORT" + +backend_pid="" +worker_pid="" +frontend_pid="" + +fail() { + printf 'launch-production-like-dev: %s\n' "$*" >&2 + exit 1 +} + +[ -f "$PROFILE_ENV" ] || fail "profile env file not found: $PROFILE_ENV" + +set -a +# shellcheck disable=SC1090 +. "$PROFILE_ENV" +set +a + +export APP_ENV="${APP_ENV:-staging}" +export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops}" +export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" +export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" +export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}" +export CELERY_ENABLED="${CELERY_ENABLED:-true}" +export CELERY_QUEUES="${CELERY_QUEUES:-send_email,append_sent,default}" +export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}" +export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$ROOT/runtime/production-like/files}" +export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}" +export DEV_BOOTSTRAP_ENABLED="${DEV_BOOTSTRAP_ENABLED:-true}" +export CORS_ORIGINS="${CORS_ORIGINS:-http://127.0.0.1:$FRONTEND_PORT,http://localhost:$FRONTEND_PORT}" + +if [ -z "${MASTER_KEY_B64:-}" ]; then + MASTER_KEY_B64="$("$PYTHON" - <<'PY' +from cryptography.fernet import Fernet +print(Fernet.generate_key().decode()) +PY +)" + export MASTER_KEY_B64 +fi + +compose() { + "$DOCKER" compose --env-file "$PROFILE_ENV" -f "$PROFILE_ROOT/docker-compose.yml" "$@" +} + +port_is_free() { + "$PYTHON" - "$1" "$2" <<'PY' +import socket +import sys + +host = sys.argv[1] +port = int(sys.argv[2]) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + sock.bind((host, port)) + except OSError: + raise SystemExit(1) +PY +} + +wait_for_tcp() { + "$PYTHON" - "$1" "$2" <<'PY' +import socket +import sys +import time + +host = sys.argv[1] +port = int(sys.argv[2]) +deadline = time.monotonic() + 90 +last_error = None +while time.monotonic() < deadline: + try: + with socket.create_connection((host, port), timeout=2): + raise SystemExit(0) + except OSError as exc: + last_error = exc + time.sleep(1) +print(f"Timed out waiting for {host}:{port}: {last_error}", file=sys.stderr) +raise SystemExit(1) +PY +} + +wait_for_url() { + "$PYTHON" - "$1" <<'PY' +import sys +import time +import urllib.request + +url = sys.argv[1] +deadline = time.monotonic() + 90 +last_error = None +while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as response: + if 200 <= response.status < 500: + raise SystemExit(0) + except Exception as exc: # noqa: BLE001 - printed only on timeout. + last_error = exc + time.sleep(1) +print(f"Timed out waiting for {url}: {last_error}", file=sys.stderr) +raise SystemExit(1) +PY +} + +cleanup() { + if [ -n "${frontend_pid:-}" ] && kill -0 "$frontend_pid" 2>/dev/null; then + kill "$frontend_pid" 2>/dev/null || true + fi + if [ -n "${backend_pid:-}" ] && kill -0 "$backend_pid" 2>/dev/null; then + kill "$backend_pid" 2>/dev/null || true + fi + if [ -n "${worker_pid:-}" ] && kill -0 "$worker_pid" 2>/dev/null; then + kill "$worker_pid" 2>/dev/null || true + fi + if [ "$STOP_DEPENDENCIES_ON_EXIT" = "1" ]; then + compose down >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT INT TERM + +[ -x "$PYTHON" ] || fail "Python virtualenv not found at $PYTHON. Run: cd $ROOT && ./.venv/bin/python -m pip install -r requirements-dev.txt" +[ -x "$NPM" ] || fail "npm not found at $NPM. Set NODE_BIN or NPM to your Node installation." +[ -f "$WEBUI_ROOT/package.json" ] || fail "WebUI package.json not found at $WEBUI_ROOT/package.json" +[ -d "$WEBUI_ROOT/node_modules" ] || fail "WebUI node_modules missing. Run: cd $WEBUI_ROOT && PATH=$NODE_BIN:\$PATH $NPM install" + +mkdir -p "$LOG_DIR" "$FILE_STORAGE_LOCAL_ROOT" +: > "$BACKEND_LOG" +: > "$WORKER_LOG" +: > "$FRONTEND_LOG" + +port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use" +port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use" + +printf 'Starting production-like dependencies through Docker Compose\n' +compose up -d + +wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_POSTGRES_PORT:-55433}" +wait_for_tcp 127.0.0.1 "${GOVOPLAN_PRODUCTION_LIKE_REDIS_PORT:-56379}" + +printf 'Running migrations and development bootstrap against %s\n' "$DATABASE_URL" +( + cd "$ROOT" + "$PYTHON" -m govoplan_core.commands.init_db --database-url "$DATABASE_URL" --with-dev-data +) + +printf 'Starting GovOPlaN backend at %s\n' "$BACKEND_URL" +( + cd "$ROOT" + "$PYTHON" -m govoplan_core.devserver --host "$BACKEND_HOST" --port "$BACKEND_PORT" +) >"$BACKEND_LOG" 2>&1 & +backend_pid="$!" + +printf 'Waiting for %s/health\n' "$BACKEND_URL" +wait_for_url "$BACKEND_URL/health" || { + tail -n 80 "$BACKEND_LOG" >&2 || true + fail "backend did not become healthy" +} + +printf 'Starting Celery worker for queues %s\n' "$CELERY_QUEUES" +( + cd "$ROOT" + "$PYTHON" -m celery -A govoplan_core.celery_app:celery worker \ + --queues "$CELERY_QUEUES" \ + --hostname "govoplan-production-like@%h" \ + --loglevel INFO +) >"$WORKER_LOG" 2>&1 & +worker_pid="$!" + +printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL" +( + cd "$WEBUI_ROOT" + PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \ + CHOKIDAR_USEPOLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" \ + CHOKIDAR_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" \ + VITE_API_PROXY_TARGET="$BACKEND_URL" \ + "$NPM" run dev -- --force +) >"$FRONTEND_LOG" 2>&1 & +frontend_pid="$!" + +printf 'Waiting for %s\n' "$FRONTEND_URL" +wait_for_url "$FRONTEND_URL" || { + tail -n 80 "$FRONTEND_LOG" >&2 || true + fail "frontend did not become reachable" +} + +if [ "$OPEN_BROWSER" = "1" ] && command -v xdg-open >/dev/null 2>&1; then + xdg-open "$FRONTEND_URL" >/dev/null 2>&1 || true +fi + +cat < int: + parser = argparse.ArgumentParser(description="Run safe module-installer rollback drills in temporary runtimes.") + parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS), help="Scenario to run. Defaults to all scenarios.") + parser.add_argument("--runtime-root", type=Path, help="Directory for temporary drill runtimes. Defaults to a new temp directory.") + parser.add_argument("--keep-runtime", action="store_true", help="Keep the temporary drill runtime after completion.") + parser.add_argument("--format", choices=("json", "text"), default="text", help="Output format.") + args = parser.parse_args() + + runtime_root = args.runtime_root or Path(tempfile.mkdtemp(prefix="govoplan-installer-drill-")) + runtime_root.mkdir(parents=True, exist_ok=True) + selected = args.scenario or sorted(SCENARIOS) + results: list[dict[str, object]] = [] + ok = True + try: + for name in selected: + scenario_root = runtime_root / name + scenario_root.mkdir(parents=True, exist_ok=True) + try: + result = SCENARIOS[name](scenario_root) + except Exception as exc: + ok = False + result = {"scenario": name, "ok": False, "error": str(exc), "root": str(scenario_root)} + else: + ok = ok and bool(result.get("ok")) + results.append(result) + finally: + if not args.keep_runtime and args.runtime_root is None: + shutil.rmtree(runtime_root, ignore_errors=True) + + payload = {"ok": ok, "runtime_root": str(runtime_root), "scenarios": results} + if args.format == "json": + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + for result in results: + status = "ok" if result.get("ok") else "FAILED" + print(f"{status}: {result['scenario']}") + if result.get("run_id"): + print(f" run: {result['run_id']}") + if result.get("record_path"): + print(f" record: {result['record_path']}") + if result.get("error"): + print(f" error: {result['error']}") + print(f"runtime: {runtime_root}") + return 0 if ok else 1 + + +def package_failure(root: Path) -> dict[str, object]: + database_url, database = _sqlite_database(root) + plan = _saved_install_plan(database, "package-failure-example") + with database.session() as session, _patch_subprocess(_package_failure_run): + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=database_url, + runtime_dir=root / "installer", + ) + restored_plan = saved_module_install_plan(session) + record = _run_record(root, result.run_id) + return _scenario_result( + "package-failure", + root, + result, + expected_status="rolled-back", + checks={ + "supervisor_status": _nested(record, "supervisor", "status") == "rolled-back", + "plan_restored": tuple(item.status for item in restored_plan.items) == ("planned",), + }, + ) + + +def migration_failure(root: Path) -> dict[str, object]: + database_url, database = _sqlite_database(root) + plan = _saved_install_plan(database, "migration-failure-example") + with database.session() as session, _patch_subprocess(_migration_failure_run): + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=database_url, + runtime_dir=root / "installer", + migrate_database=True, + ) + record = _run_record(root, result.run_id) + return _scenario_result( + "migration-failure", + root, + result, + expected_status="rolled-back", + checks={ + "sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict), + "supervisor_status": _nested(record, "supervisor", "status") == "rolled-back", + }, + ) + + +def restart_failure(root: Path) -> dict[str, object]: + database_url, database = _sqlite_database(root) + plan = _saved_install_plan(database, "restart-failure-example") + with database.session() as session, _patch_subprocess(_restart_failure_run): + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=database_url, + runtime_dir=root / "installer", + restart_command="restart-fails", + ) + record = _run_record(root, result.run_id) + return _scenario_result( + "restart-failure", + root, + result, + expected_status="rolled-back", + checks={"supervisor_status": _nested(record, "supervisor", "status") == "rolled-back"}, + ) + + +def health_timeout(root: Path) -> dict[str, object]: + database_url, database = _sqlite_database(root) + plan = _saved_install_plan(database, "health-timeout-example") + with database.session() as session, _patch_subprocess(_success_run), _patch_urlopen(): + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=database_url, + runtime_dir=root / "installer", + restart_command="restart-ok", + health_url="http://127.0.0.1:9/health", + health_timeout_seconds=0.2, + health_interval_seconds=0.05, + ) + record = _run_record(root, result.run_id) + return _scenario_result( + "health-timeout", + root, + result, + expected_status="rolled-back", + checks={ + "health_failure_recorded": "127.0.0.1:9/health" in str(_nested(record, "supervisor", "failure_reason")), + "supervisor_status": _nested(record, "supervisor", "status") == "rolled-back", + }, + ) + + +def destructive_retirement_failure(root: Path) -> dict[str, object]: + database_url, database = _sqlite_database(root) + available = {"retirement-example": _retirement_failure_manifest()} + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "retirement-example", + "action": "uninstall", + "python_package": "govoplan-retirement-example", + "destroy_data": True, + }]) + session.commit() + with _patch_subprocess(_success_run): + result = run_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=database_url, + runtime_dir=root / "installer", + ) + record = _run_record(root, result.run_id) + return _scenario_result( + "destructive-retirement-failure", + root, + result, + expected_status="rolled-back", + checks={ + "rollback_recorded": isinstance(record.get("destructive_retirement_rollback"), dict), + "sqlite_backup_recorded": isinstance(_nested(record, "snapshot", "database_backup"), dict), + }, + ) + + +def postgres_hooks(root: Path) -> dict[str, object]: + _, database = _sqlite_database(root) + backup_script = ( + "import json, os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); " + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')" + ) + check_script = ( + "import os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text(" + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')" + ) + restore_script = ( + "import os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text(" + "os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')" + ) + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "postgres-hook-example", + "action": "install", + "python_package": "govoplan-postgres-hook-example", + "python_ref": "govoplan-postgres-hook-example==0.1.0", + }]) + session.commit() + result = run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url="postgresql://db.example.invalid/govoplan", + runtime_dir=root / "installer", + migrate_database=True, + database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}", + database_restore_command=f"{sys.executable} -c {shlex.quote(restore_script)}", + database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}", + dry_run=True, + ) + with _patch_subprocess(_non_shell_success_run): + rollback = rollback_module_install_run( + run_id=result.run_id, + runtime_dir=root / "installer", + database_url="postgresql://db.example.invalid/govoplan", + ) + run_dir = result.record_path.parent + record = _run_record(root, result.run_id) + ok = ( + result.status == "dry-run" + and rollback.status == "rolled-back" + and (run_dir / "restore-check.marker").read_text(encoding="utf-8") == "backup" + and (run_dir / "restore.marker").read_text(encoding="utf-8") == "postgresql://db.example.invalid/govoplan" + and isinstance(_nested(record, "snapshot", "database_backup"), dict) + ) + return { + "scenario": "postgres-hooks", + "ok": ok, + "root": str(root), + "run_id": result.run_id, + "record_path": str(result.record_path), + "rollback_status": rollback.status, + } + + +def queue_and_lock(root: Path) -> dict[str, object]: + runtime_dir = root / "installer" + status = update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={"status": "polling", "current_request_id": None}, + ) + request = queue_module_installer_request( + runtime_dir=runtime_dir, + requested_by="drill", + options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]}, + ) + claimed = claim_next_module_installer_request(runtime_dir=runtime_dir) + assert claimed is not None + failed = update_module_installer_request( + runtime_dir=runtime_dir, + request_id=str(claimed["request_id"]), + patch={"status": "failed", "error": "simulated drill failure"}, + ) + retry = retry_module_installer_request(runtime_dir=runtime_dir, request_id=str(failed["request_id"]), requested_by="drill") + cancelled = cancel_module_installer_request(runtime_dir=runtime_dir, request_id=str(retry["request_id"]), cancelled_by="drill") + lock_path = runtime_dir / "install.lock" + lock_path.write_text(json.dumps({"pid": 999999, "created_at": "drill"}), encoding="utf-8") + lock = module_installer_lock_status(runtime_dir=runtime_dir) + lock_path.unlink() + return { + "scenario": "queue-and-lock", + "ok": bool(status["running"]) and request["status"] == "queued" and failed["status"] == "failed" and cancelled["status"] == "cancelled" and lock["locked"] is True and not module_installer_lock_status(runtime_dir=runtime_dir)["locked"], + "root": str(root), + "daemon_status_path": str(runtime_dir / "daemon.status.json"), + } + + +def _sqlite_database(root: Path) -> tuple[str, Any]: + database_url = f"sqlite:///{root / 'drill.db'}" + configure_database(database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + return database_url, database + + +def _saved_install_plan(database: Any, module_id: str) -> ModuleInstallPlan: + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": module_id, + "action": "install", + "python_package": f"govoplan-{module_id}", + "python_ref": f"govoplan-{module_id}==0.1.0", + }]) + save_desired_enabled_modules(session, ("tenancy", "access")) + session.commit() + return plan + + +def _retirement_failure_manifest() -> ModuleManifest: + def destroy(_session: object, _module_id: str) -> None: + raise RuntimeError("simulated destructive retirement failure") + + def provider(_session: object | None, _module_id: str) -> MigrationRetirementPlan: + return MigrationRetirementPlan( + supported=True, + summary="Drill retirement provider supports destructive cleanup.", + destroy_data_supported=True, + destroy_data_summary="Drill destructive cleanup will fail during execution.", + destroy_data_executor=destroy, + ) + + return ModuleManifest( + id="retirement-example", + name="Retirement example", + version="0.1.0", + migration_spec=MigrationSpec( + module_id="retirement-example", + retirement_supported=True, + retirement_provider=provider, + ), + ) + + +def _scenario_result( + scenario: str, + root: Path, + result: Any, + *, + expected_status: str, + checks: dict[str, bool], +) -> dict[str, object]: + return { + "scenario": scenario, + "ok": result.status == expected_status and all(checks.values()), + "root": str(root), + "run_id": result.run_id, + "record_path": str(result.record_path), + "status": result.status, + "checks": checks, + } + + +def _run_record(root: Path, run_id: str) -> dict[str, object]: + return read_module_installer_run(runtime_dir=root / "installer", run_id=run_id) + + +def _nested(value: dict[str, object], *keys: str) -> object: + current: object = value + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + return current + + +def _command_text(argv: object) -> str: + if isinstance(argv, list | tuple): + return " ".join(str(item) for item in argv) + return str(argv) + + +def _completed(return_code: int = 0, stdout: str = "", stderr: str = "") -> SimpleNamespace: + return SimpleNamespace(returncode=return_code, stdout=stdout, stderr=stderr) + + +def _success_run(argv: object, *_args: object, **_kwargs: object) -> SimpleNamespace: + text = _command_text(argv) + if "pip freeze" in text: + return _completed(stdout="govoplan-core==0.0.0\n") + return _completed() + + +def _non_shell_success_run(argv: object, *_args: object, **kwargs: object) -> Any: + if kwargs.get("shell"): + return ORIGINAL_SUBPROCESS_RUN(argv, *_args, **kwargs) + return _success_run(argv, *_args, **kwargs) + + +def _package_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace: + text = _command_text(argv) + if "pip install" in text and "==" in text and "-r" not in text: + return _completed(23, stderr="simulated package install failure") + return _success_run(argv, *_args, **kwargs) + + +def _migration_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace: + text = _command_text(argv) + if "govoplan_core.commands.init_db" in text: + return _completed(24, stderr="simulated migration failure") + return _success_run(argv, *_args, **kwargs) + + +def _restart_failure_run(argv: object, *_args: object, **kwargs: object) -> SimpleNamespace: + if kwargs.get("shell") and str(argv).strip() == "restart-fails": + return _completed(25, stderr="simulated restart failure") + return _success_run(argv, *_args, **kwargs) + + +ORIGINAL_SUBPROCESS_RUN = subprocess.run + + +@contextmanager +def _patch_subprocess(fake_run: Callable[..., Any]) -> Iterable[None]: + import govoplan_core.core.module_installer as installer + + original = installer.subprocess.run + installer.subprocess.run = fake_run # type: ignore[assignment] + try: + yield + finally: + installer.subprocess.run = original # type: ignore[assignment] + + +@contextmanager +def _patch_urlopen() -> Iterable[None]: + import govoplan_core.core.module_installer as installer + + original = installer.urllib.request.urlopen + + def fail(*_args: object, **_kwargs: object) -> None: + raise URLError("simulated health timeout") + + installer.urllib.request.urlopen = fail # type: ignore[assignment] + try: + yield + finally: + installer.urllib.request.urlopen = original # type: ignore[assignment] + + +SCENARIOS: dict[str, Scenario] = { + "destructive-retirement-failure": destructive_retirement_failure, + "health-timeout": health_timeout, + "migration-failure": migration_failure, + "package-failure": package_failure, + "postgres-hooks": postgres_hooks, + "queue-and-lock": queue_and_lock, + "restart-failure": restart_failure, +} + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/postgres-integration-check.py b/scripts/postgres-integration-check.py new file mode 100644 index 0000000..80b57ee --- /dev/null +++ b/scripts/postgres-integration-check.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +from __future__ import annotations + +import argparse +import base64 +import os +from pathlib import Path +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" + +DEFAULT_MODULE_SETS: tuple[tuple[str, str], ...] = ( + ("core-only", "tenancy,access,admin"), + ("files-only", "tenancy,access,admin,files"), + ("mail-only", "tenancy,access,admin,mail"), + ("campaign-only", "tenancy,access,admin,campaigns"), + ("campaign-with-files", "tenancy,access,admin,campaigns,files"), + ("campaign-with-mail", "tenancy,access,admin,campaigns,mail"), + ("full-product", "tenancy,access,admin,policy,audit,campaigns,files,mail,calendar,docs,ops"), +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run GovOPlaN migrations and startup smoke checks against a disposable PostgreSQL database.", + ) + parser.add_argument( + "--database-url", + default=os.environ.get("GOVOPLAN_POSTGRES_DATABASE_URL") or os.environ.get("DATABASE_URL"), + help="SQLAlchemy PostgreSQL URL, for example postgresql+psycopg://user:pass@127.0.0.1:55432/govoplan.", + ) + parser.add_argument( + "--module-set", + action="append", + default=[], + metavar="NAME=MODULES", + help="Module permutation to check. May be passed multiple times. Defaults to the standard product permutations.", + ) + parser.add_argument( + "--reset-schema", + action="store_true", + help="DROP and recreate the public schema before every module set. Use only with a disposable database.", + ) + parser.add_argument("--skip-migrate", action="store_true", help="Skip govoplan_core.commands.init_db.") + parser.add_argument("--skip-smoke", action="store_true", help="Skip govoplan_core.devserver --smoke.") + parser.add_argument("--python", default=sys.executable, help="Python executable to use. Defaults to the current interpreter.") + parser.add_argument("--app-env", default="staging", help="APP_ENV value to pass to subprocesses. Defaults to staging.") + args = parser.parse_args() + + database_url = (args.database_url or "").strip() + if not database_url: + parser.error("--database-url or GOVOPLAN_POSTGRES_DATABASE_URL is required") + if not database_url.startswith("postgresql"): + parser.error("This check is PostgreSQL-only. Use a postgresql+psycopg:// or postgresql:// DATABASE_URL.") + + module_sets = _parse_module_sets(args.module_set) + ok = True + for name, modules in module_sets: + print(f"\n== {name}: {modules} ==") + if args.reset_schema: + _reset_public_schema(database_url) + env = _check_env(database_url=database_url, modules=modules, app_env=args.app_env) + if not args.skip_migrate: + ok = _run([args.python, "-m", "govoplan_core.commands.init_db", "--database-url", database_url], env=env) and ok + if not args.skip_smoke: + ok = _run([args.python, "-m", "govoplan_core.devserver", "--smoke", "--no-reload"], env=env) and ok + return 0 if ok else 1 + + +def _parse_module_sets(raw: list[str]) -> tuple[tuple[str, str], ...]: + if not raw: + return DEFAULT_MODULE_SETS + parsed: list[tuple[str, str]] = [] + for item in raw: + name, separator, modules = item.partition("=") + if not separator or not name.strip() or not modules.strip(): + raise SystemExit(f"Invalid --module-set value {item!r}. Use NAME=module_a,module_b.") + parsed.append((name.strip(), modules.strip())) + return tuple(parsed) + + +def _check_env(*, database_url: str, modules: str, app_env: str) -> dict[str, str]: + env = os.environ.copy() + env["APP_ENV"] = app_env + env["DATABASE_URL"] = database_url + env["ENABLED_MODULES"] = modules + env["DEV_AUTO_MIGRATE_ENABLED"] = "false" + env["DEV_BOOTSTRAP_ENABLED"] = "false" + env["CELERY_ENABLED"] = "false" + env.setdefault("MASTER_KEY_B64", base64.urlsafe_b64encode(os.urandom(32)).decode("ascii")) + env.setdefault("FILE_STORAGE_BACKEND", "local") + env.setdefault("FILE_STORAGE_LOCAL_ROOT", str(ROOT / "runtime" / "postgres-check-files")) + env.setdefault("MOCK_MAILBOX_DIR", str(ROOT / "runtime" / "postgres-check-mock-mailbox")) + env["PYTHONPATH"] = os.pathsep.join(part for part in (str(SRC), env.get("PYTHONPATH", "")) if part) + return env + + +def _run(command: list[str], *, env: dict[str, str]) -> bool: + print("+ " + " ".join(command)) + result = subprocess.run(command, cwd=ROOT, env=env, check=False) + if result.returncode != 0: + print(f"Command failed with exit code {result.returncode}: {' '.join(command)}", file=sys.stderr) + return False + return True + + +def _reset_public_schema(database_url: str) -> None: + try: + from sqlalchemy import create_engine + except ModuleNotFoundError as exc: + raise SystemExit("SQLAlchemy is required for --reset-schema. Install govoplan-core requirements first.") from exc + + engine = create_engine(database_url, isolation_level="AUTOCOMMIT") + try: + with engine.connect() as connection: + connection.exec_driver_sql("DROP SCHEMA IF EXISTS public CASCADE") + connection.exec_driver_sql("CREATE SCHEMA public") + connection.exec_driver_sql("GRANT ALL ON SCHEMA public TO public") + finally: + engine.dispose() + print("Reset PostgreSQL schema: public") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/push-release-tag.sh b/scripts/push-release-tag.sh index 7e3a732..715c0b0 100644 --- a/scripts/push-release-tag.sh +++ b/scripts/push-release-tag.sh @@ -24,6 +24,11 @@ Options: -m, --message Commit message. Defaults to "Release v". --tag-message Annotated tag message. Defaults to the commit message. --skip-release-lock Do not regenerate core webui/package-lock.release.json. + --warn-migration-audit Run the migration release audit without baseline strictness. + --strict-migration-audit + Require current Alembic heads to be recorded in the + latest migration release baseline. + --skip-migration-audit Do not run the release migration audit. --publish-web-catalog After core/modules are pushed, publish a signed release catalog into govoplan-web. --web-root govoplan-web checkout for --publish-web-catalog. @@ -51,18 +56,22 @@ Repos: govoplan-access govoplan-admin govoplan-tenancy + govoplan-organizations + govoplan-identity govoplan-policy govoplan-audit + govoplan-dashboard govoplan-files govoplan-mail govoplan-campaign govoplan-calendar + govoplan-ops govoplan-core Roadmap/scaffold module repositories are tag-only until they have package metadata: addresses, appointments, cases, connectors, dms, erp, - fit-connect, forms, identity-trust, idm, ledger, notifications, ops, - payments, portal, reporting, scheduling, search, tasks, templates, workflow, xoev, + fit-connect, forms, identity-trust, idm, ledger, notifications, payments, + portal, reporting, scheduling, search, tasks, templates, workflow, xoev, xrechnung, and xta-osci. Examples: @@ -83,12 +92,16 @@ PACKAGE_MODULE_REPOS=( "$PARENT/govoplan-access" "$PARENT/govoplan-admin" "$PARENT/govoplan-tenancy" + "$PARENT/govoplan-organizations" + "$PARENT/govoplan-identity" "$PARENT/govoplan-policy" "$PARENT/govoplan-audit" + "$PARENT/govoplan-dashboard" "$PARENT/govoplan-files" "$PARENT/govoplan-mail" "$PARENT/govoplan-campaign" "$PARENT/govoplan-calendar" + "$PARENT/govoplan-ops" ) TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-addresses" @@ -103,7 +116,6 @@ TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-idm" "$PARENT/govoplan-ledger" "$PARENT/govoplan-notifications" - "$PARENT/govoplan-ops" "$PARENT/govoplan-payments" "$PARENT/govoplan-portal" "$PARENT/govoplan-reporting" @@ -132,6 +144,7 @@ TAG_MESSAGE="" DRY_RUN=0 YES=0 GENERATE_RELEASE_LOCK=1 +MIGRATION_AUDIT_MODE="auto" NPM_BIN="${NPM:-}" PUBLISH_WEB_CATALOG=0 WEB_ROOT="$PARENT/govoplan-web" @@ -282,6 +295,12 @@ update_manifest_version() { govoplan-tenancy) manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py" ;; + govoplan-organizations) + manifest_path="$repo/src/govoplan_organizations/backend/manifest.py" + ;; + govoplan-identity) + manifest_path="$repo/src/govoplan_identity/backend/manifest.py" + ;; govoplan-policy) manifest_path="$repo/src/govoplan_policy/backend/manifest.py" ;; @@ -408,6 +427,32 @@ generate_release_lock() { run "$ROOT/scripts/generate-release-lock.sh" --npm "$NPM_BIN" } +run_migration_release_audit() { + local audit_script="$ROOT/scripts/release-migration-audit.py" + local command=("$PYTHON" "$audit_script") + + case "$MIGRATION_AUDIT_MODE" in + skip) + echo "Migration release audit: skipped" + return + ;; + strict) + command+=("--strict") + ;; + auto) + command+=("--strict-if-baseline") + ;; + warn) + ;; + *) + fail "unknown migration audit mode: $MIGRATION_AUDIT_MODE" + ;; + esac + + [[ -f "$audit_script" ]] || fail "missing migration audit helper: $audit_script" + run "${command[@]}" +} + print_command() { printf '+' printf ' %q' "$@" @@ -514,6 +559,18 @@ while [[ $# -gt 0 ]]; do GENERATE_RELEASE_LOCK=0 shift ;; + --warn-migration-audit) + MIGRATION_AUDIT_MODE="warn" + shift + ;; + --strict-migration-audit) + MIGRATION_AUDIT_MODE="strict" + shift + ;; + --skip-migration-audit) + MIGRATION_AUDIT_MODE="skip" + shift + ;; --publish-web-catalog) PUBLISH_WEB_CATALOG=1 shift @@ -690,6 +747,7 @@ if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then else echo " release lock: skipped" fi +echo " migration audit: $MIGRATION_AUDIT_MODE" if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed" else @@ -707,6 +765,8 @@ for repo in "${REPOS[@]}"; do echo done +run_migration_release_audit + confirm_release for repo in "${PACKAGE_REPOS[@]}"; do diff --git a/scripts/release-migration-audit.py b/scripts/release-migration-audit.py new file mode 100644 index 0000000..66f0df8 --- /dev/null +++ b/scripts/release-migration-audit.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python +from __future__ import annotations + +import argparse +import ast +from dataclasses import dataclass +from datetime import datetime, timezone +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BASELINE_FILE = ROOT / "docs" / "migration-release-baselines.json" + + +@dataclass(frozen=True, slots=True) +class Migration: + owner: str + path: Path + revision: str + down_revisions: tuple[str, ...] + branch_labels: tuple[str, ...] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Audit GovOPlaN Alembic migrations against the release baseline policy.", + ) + parser.add_argument( + "--workspace-root", + type=Path, + default=ROOT.parent, + help="Directory containing govoplan-* checkouts. Defaults to the parent of govoplan-core.", + ) + parser.add_argument( + "--baseline-file", + type=Path, + default=DEFAULT_BASELINE_FILE, + help="JSON file recording released migration heads.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Fail when current heads are not recorded in the latest release baseline.", + ) + parser.add_argument( + "--strict-if-baseline", + action="store_true", + help="Run in strict mode only after at least one release baseline is recorded.", + ) + parser.add_argument( + "--record-release", + metavar="VERSION", + help="Record the current reviewed migration heads as a release baseline.", + ) + parser.add_argument( + "--replace-release", + action="store_true", + help="Replace an existing release entry when used with --record-release.", + ) + parser.add_argument( + "--squash-plan", + action="store_true", + help="Print the reviewed/manual squash checklist for the current graph.", + ) + parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.") + args = parser.parse_args() + + migrations = discover_migrations(args.workspace_root) + baseline = load_baseline_file(args.baseline_file) + report = build_report( + migrations, + baseline=baseline, + baseline_file=args.baseline_file, + workspace_root=args.workspace_root, + ) + + if args.record_release: + if report["graph_errors"]: + print_text_report(report) + return 1 + baseline = record_release_baseline( + baseline, + report, + release=args.record_release, + replace=args.replace_release, + ) + write_baseline_file(args.baseline_file, baseline) + report = build_report( + migrations, + baseline=baseline, + baseline_file=args.baseline_file, + workspace_root=args.workspace_root, + ) + + if args.squash_plan: + print_squash_plan(report) + elif args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + print_text_report(report) + + has_graph_errors = bool(report["graph_errors"]) + has_strict_errors = bool(report["strict_errors"]) + strict_required = args.strict or (args.strict_if_baseline and report["release_count"] > 0) + if has_graph_errors or (strict_required and has_strict_errors): + return 1 + return 0 + + +def discover_migrations(workspace_root: Path) -> list[Migration]: + migrations: list[Migration] = [] + for versions_dir in _migration_version_dirs(workspace_root): + owner = owner_for_versions_dir(versions_dir) + for path in sorted(versions_dir.glob("*.py")): + if path.name == "__init__.py": + continue + migration = parse_migration_file(owner, path) + if migration is not None: + migrations.append(migration) + return migrations + + +def _migration_version_dirs(workspace_root: Path) -> list[Path]: + candidates = [ROOT / "alembic" / "versions"] + for repo in sorted(workspace_root.glob("govoplan-*")): + candidates.extend(sorted(repo.glob("src/govoplan_*/backend/migrations/versions"))) + seen: set[Path] = set() + result: list[Path] = [] + for candidate in candidates: + resolved = candidate.resolve() + if resolved in seen or not candidate.is_dir(): + continue + seen.add(resolved) + result.append(candidate) + return result + + +def owner_for_versions_dir(versions_dir: Path) -> str: + if (ROOT / "alembic" / "versions").resolve() == versions_dir.resolve(): + return "govoplan-core" + for parent in versions_dir.parents: + if parent.name.startswith("govoplan-"): + return parent.name + return versions_dir.parent.name + + +def parse_migration_file(owner: str, path: Path) -> Migration | None: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + values: dict[str, Any] = {} + for statement in tree.body: + if isinstance(statement, ast.Assign): + for target in statement.targets: + if isinstance(target, ast.Name) and target.id in {"revision", "down_revision", "branch_labels"}: + values[target.id] = ast.literal_eval(statement.value) + elif ( + isinstance(statement, ast.AnnAssign) + and isinstance(statement.target, ast.Name) + and statement.target.id in {"revision", "down_revision", "branch_labels"} + and statement.value is not None + ): + values[statement.target.id] = ast.literal_eval(statement.value) + revision = values.get("revision") + if not isinstance(revision, str): + return None + return Migration( + owner=owner, + path=path, + revision=revision, + down_revisions=_normalize_revision_tuple(values.get("down_revision")), + branch_labels=_normalize_string_tuple(values.get("branch_labels")), + ) + + +def _normalize_revision_tuple(value: Any) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value,) + if isinstance(value, (list, tuple, set)): + return tuple(str(item) for item in value if item is not None) + return (str(value),) + + +def _normalize_string_tuple(value: Any) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value,) + if isinstance(value, (list, tuple, set)): + return tuple(str(item) for item in value) + return (str(value),) + + +def load_baseline_file(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"version": 1, "releases": [], "_missing": True} + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise SystemExit(f"{path} must contain a JSON object") + releases = payload.get("releases") + if not isinstance(releases, list): + raise SystemExit(f"{path} must contain a releases list") + return payload + + +def build_report( + migrations: list[Migration], + *, + baseline: dict[str, Any], + baseline_file: Path, + workspace_root: Path, +) -> dict[str, Any]: + revisions: dict[str, Migration] = {} + graph_errors: list[str] = [] + for migration in migrations: + existing = revisions.get(migration.revision) + if existing is not None: + graph_errors.append( + f"duplicate revision {migration.revision}: {existing.path} and {migration.path}" + ) + revisions[migration.revision] = migration + + referenced = {revision for migration in migrations for revision in migration.down_revisions} + for migration in migrations: + for down_revision in migration.down_revisions: + if down_revision not in revisions: + graph_errors.append( + f"{migration.revision} references missing down_revision {down_revision} in {migration.path}" + ) + + current_heads = sorted(revision for revision in revisions if revision not in referenced) + current_head_entries = [ + { + "owner": revisions[revision].owner, + "revision": revision, + } + for revision in current_heads + ] + owner_rows = [] + for owner in sorted({migration.owner for migration in migrations}): + owner_migrations = [migration for migration in migrations if migration.owner == owner] + owner_revisions = {migration.revision for migration in owner_migrations} + owner_referenced = { + revision + for migration in owner_migrations + for revision in migration.down_revisions + if revision in owner_revisions + } + owner_heads = sorted(owner_revisions - owner_referenced) + owner_rows.append( + { + "owner": owner, + "migrations": len(owner_migrations), + "heads": owner_heads, + } + ) + + releases = baseline.get("releases") or [] + latest_release = releases[-1] if releases else None + latest_heads = _latest_release_heads(latest_release) + unrecorded_heads = sorted(set(current_heads) - latest_heads) if latest_release else current_heads + + strict_errors: list[str] = [] + if baseline.get("_missing"): + strict_errors.append(f"baseline file is missing: {baseline_file}") + if not releases: + strict_errors.append("no released migration baseline has been recorded yet") + if unrecorded_heads: + strict_errors.append( + "current migration heads are not recorded in the latest release baseline: " + + ", ".join(unrecorded_heads) + ) + return { + "workspace_root": str(workspace_root), + "baseline_file": str(baseline_file), + "baseline_file_missing": bool(baseline.get("_missing")), + "release_count": len(releases), + "latest_release": latest_release, + "owners": owner_rows, + "current_heads": current_heads, + "current_head_entries": current_head_entries, + "graph_errors": graph_errors, + "strict_errors": strict_errors, + } + + +def _latest_release_heads(latest_release: Any) -> set[str]: + if not isinstance(latest_release, dict): + return set() + heads = latest_release.get("heads") or [] + graph_heads = latest_release.get("graph_heads") or [] + result: set[str] = set() + for head_list in (heads, graph_heads): + if not isinstance(head_list, list): + continue + for entry in head_list: + if isinstance(entry, str): + result.add(entry) + elif isinstance(entry, dict) and isinstance(entry.get("revision"), str): + result.add(entry["revision"]) + owner_heads = latest_release.get("owner_heads") or [] + if isinstance(owner_heads, list): + for entry in owner_heads: + if not isinstance(entry, dict): + continue + revisions = entry.get("revisions") or [] + if isinstance(revisions, str): + result.add(revisions) + elif isinstance(revisions, list): + result.update(revision for revision in revisions if isinstance(revision, str)) + return result + + +def record_release_baseline( + baseline: dict[str, Any], + report: dict[str, Any], + *, + release: str, + replace: bool, +) -> dict[str, Any]: + payload = { + key: value + for key, value in baseline.items() + if not key.startswith("_") + } + payload.setdefault("version", 1) + releases = list(payload.get("releases") or []) + existing_index = next( + (index for index, item in enumerate(releases) if isinstance(item, dict) and item.get("release") == release), + None, + ) + if existing_index is not None and not replace: + raise SystemExit(f"release {release} already exists in baseline file; pass --replace-release to update it") + + entry = { + "release": release, + "recorded_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + "squash_policy": "reviewed-manual", + "heads": report["current_head_entries"], + "owner_heads": [ + { + "owner": owner["owner"], + "revisions": owner["heads"], + } + for owner in report["owners"] + ], + } + if existing_index is None: + releases.append(entry) + else: + releases[existing_index] = entry + payload["releases"] = releases + return payload + + +def write_baseline_file(path: Path, baseline: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def print_text_report(report: dict[str, Any]) -> None: + print("Migration release audit") + print(f" workspace: {report['workspace_root']}") + print(f" baseline file: {report['baseline_file']}") + print(f" recorded releases: {report['release_count']}") + print() + print("Owners:") + for owner in report["owners"]: + heads = ", ".join(owner["heads"]) if owner["heads"] else "-" + print(f" {owner['owner']}: {owner['migrations']} migration(s), head(s): {heads}") + print() + print("Current heads:") + for entry in report["current_head_entries"]: + print(f" {entry['owner']}: {entry['revision']}") + if not report["current_head_entries"]: + print(" -") + + if report["graph_errors"]: + print() + print("Graph errors:") + for error in report["graph_errors"]: + print(f" - {error}") + + if report["strict_errors"]: + print() + print("Release baseline warnings:") + for error in report["strict_errors"]: + print(f" - {error}") + + +def print_squash_plan(report: dict[str, Any]) -> None: + print("Manual migration squash plan") + print() + print("Policy:") + print(" - Do not rewrite revision IDs that have shipped to real installations.") + print(" - Squash only unreleased development migrations.") + print(" - Review generated baseline/upgrade migrations like normal schema code.") + print(" - Run PostgreSQL migration smoke checks after any squash.") + print() + print("Current owner heads:") + for owner in report["owners"]: + heads = ", ".join(owner["heads"]) if owner["heads"] else "-" + print(f" - {owner['owner']}: {heads}") + print() + print("Current Alembic graph heads:") + for entry in report["current_head_entries"]: + print(f" - {entry['owner']}: {entry['revision']}") + if not report["current_head_entries"]: + print(" -") + print() + print("Release steps:") + print(" 1. Decide which migration revisions are unreleased and may be folded.") + print(" 2. Replace those development chains with reviewed baseline/upgrade migrations.") + print(" 3. Run scripts/release-migration-audit.py and PostgreSQL release checks.") + print(" 4. Record the reviewed heads with scripts/release-migration-audit.py --record-release .") + print(" 5. Cut the release with scripts/push-release-tag.sh; audit is strict after the first baseline.") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/govoplan_core/admin/models.py b/src/govoplan_core/admin/models.py index 96297f7..803c7ba 100644 --- a/src/govoplan_core/admin/models.py +++ b/src/govoplan_core/admin/models.py @@ -9,7 +9,7 @@ from govoplan_core.db.base import Base, TimestampMixin class SystemSettings(Base, TimestampMixin): - __tablename__ = "system_settings" + __tablename__ = "core_system_settings" id: Mapped[str] = mapped_column(String(36), primary_key=True, default="global") default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False) diff --git a/src/govoplan_core/api/v1/schemas.py b/src/govoplan_core/api/v1/schemas.py index 027776b..cbc2022 100644 --- a/src/govoplan_core/api/v1/schemas.py +++ b/src/govoplan_core/api/v1/schemas.py @@ -24,6 +24,21 @@ class AuditLogListResponse(BaseModel): items: list[AuditLogItemResponse] +class DeltaDeletedItem(BaseModel): + id: str + resource_type: str | None = None + revision: str | None = None + deleted_at: datetime | None = None + + +class DeltaCollectionResponse(BaseModel): + items: list[dict[str, Any]] = Field(default_factory=list) + deleted: list[DeltaDeletedItem] = Field(default_factory=list) + watermark: str | None = None + has_more: bool = False + full: bool = False + + class LoginRequest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -46,6 +61,7 @@ class TenantInfo(BaseModel): name: str is_active: bool = True default_locale: str = "en" + enabled_language_codes: list[str] = Field(default_factory=list) class TenantMembershipInfo(TenantInfo): @@ -53,6 +69,16 @@ class TenantMembershipInfo(TenantInfo): is_active: bool = True +class UserUiPreferences(BaseModel): + model_config = ConfigDict(extra="ignore") + + compact_tables: bool = False + show_inline_help_hints: bool = True + reduce_motion: bool = False + sticky_section_sidebars: bool = True + theme: Literal["system", "light", "dark"] = "system" + + class UserInfo(BaseModel): id: str account_id: str @@ -63,6 +89,9 @@ class UserInfo(BaseModel): tenant_display_name: str | None = None is_tenant_admin: bool = False password_reset_required: bool = False + preferred_language: str | None = None + enabled_language_codes: list[str] = Field(default_factory=list) + ui_preferences: UserUiPreferences = Field(default_factory=UserUiPreferences) class ProfileUpdateRequest(BaseModel): @@ -70,6 +99,15 @@ class ProfileUpdateRequest(BaseModel): display_name: str | None = Field(default=None, max_length=255) tenant_display_name: str | None = Field(default=None, max_length=255) + preferred_language: str | None = Field(default=None, max_length=20) + enabled_language_codes: list[str] | None = None + ui_preferences: UserUiPreferences | None = None + + +class LanguageInfo(BaseModel): + code: str + label: str + native_label: str | None = None class RoleInfo(BaseModel): @@ -86,6 +124,25 @@ class GroupInfo(BaseModel): name: str +class PrincipalContextInfo(BaseModel): + account_id: str + membership_id: str | None = None + tenant_id: str | None = None + identity_id: str | None = None + scopes: list[str] = Field(default_factory=list) + group_ids: list[str] = Field(default_factory=list) + role_ids: list[str] = Field(default_factory=list) + function_assignment_ids: list[str] = Field(default_factory=list) + delegation_ids: list[str] = Field(default_factory=list) + auth_method: Literal["session", "api_key", "service_account"] = "session" + api_key_id: str | None = None + session_id: str | None = None + service_account_id: str | None = None + acting_for_account_id: str | None = None + email: str | None = None + display_name: str | None = None + + class LoginResponse(BaseModel): access_token: str token_type: str = "bearer" @@ -98,6 +155,10 @@ class LoginResponse(BaseModel): scopes: list[str] roles: list[RoleInfo] = Field(default_factory=list) groups: list[GroupInfo] = Field(default_factory=list) + principal: PrincipalContextInfo | None = None + available_languages: list[LanguageInfo] = Field(default_factory=list) + enabled_language_codes: list[str] = Field(default_factory=list) + default_language: str = "en" class MeResponse(BaseModel): @@ -109,3 +170,7 @@ class MeResponse(BaseModel): scopes: list[str] roles: list[RoleInfo] = Field(default_factory=list) groups: list[GroupInfo] = Field(default_factory=list) + principal: PrincipalContextInfo | None = None + available_languages: list[LanguageInfo] = Field(default_factory=list) + enabled_language_codes: list[str] = Field(default_factory=list) + default_language: str = "en" diff --git a/src/govoplan_core/audit/logging.py b/src/govoplan_core/audit/logging.py index a5bc719..29ff7f4 100644 --- a/src/govoplan_core/audit/logging.py +++ b/src/govoplan_core/audit/logging.py @@ -1,14 +1,21 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Any from sqlalchemy.orm import Session -from govoplan_access.backend.auth.dependencies import ApiPrincipal +from govoplan_access.auth import ApiPrincipal +from govoplan_core.core.change_sequence import record_change +from govoplan_core.core.events import current_event_trace, normalize_trace_id from govoplan_core.privacy.retention import sanitize_audit_details_for_policy from govoplan_audit.backend.db.models import AuditLog +AUDIT_MODULE_ID = "audit" +AUDIT_TENANT_EVENTS_COLLECTION = "audit.admin.tenant_events" +AUDIT_SYSTEM_EVENTS_COLLECTION = "audit.admin.system_events" + SENSITIVE_DETAIL_KEYS = { "password", "smtp_password", @@ -16,9 +23,40 @@ SENSITIVE_DETAIL_KEYS = { "secret", "api_key", "token", + "access_token", + "refresh_token", + "authorization", + "cookie", + "credentials", + "credential", + "private_key", + "claim_token", + "build_token", } +def _optional_text(value: object | None) -> str | None: + if value is None: + return None + candidate = str(value).strip() + return candidate or None + + +def _compact_trace(trace: Mapping[str, object] | None) -> dict[str, str]: + if not isinstance(trace, Mapping): + return {} + compact: dict[str, str] = {} + correlation_id = _optional_text(trace.get("correlation_id")) + causation_id = _optional_text(trace.get("causation_id")) + clean_correlation_id = normalize_trace_id(correlation_id) + clean_causation_id = normalize_trace_id(causation_id) + if clean_correlation_id: + compact["correlation_id"] = clean_correlation_id + if clean_causation_id: + compact["causation_id"] = clean_causation_id + return compact + + def _sanitize_details(value: Any) -> Any: if isinstance(value, dict): sanitized: dict[str, Any] = {} @@ -33,6 +71,70 @@ def _sanitize_details(value: Any) -> Any: return value +def audit_operation_context( + *, + module_id: object | None = None, + request_id: object | None = None, + run_id: object | None = None, + outcome: object | None = None, + trace: Mapping[str, object] | None = None, + **details: Any, +) -> dict[str, Any]: + """Build concise operational audit details for admin and lifecycle actions.""" + + payload: dict[str, Any] = {} + for key, value in ( + ("module_id", module_id), + ("request_id", request_id), + ("run_id", run_id), + ("outcome", outcome), + ): + text = _optional_text(value) + if text is not None: + payload[key] = text + for key, value in details.items(): + if value is not None: + payload[key] = _sanitize_details(value) + compact_trace = _compact_trace(trace) + if compact_trace: + existing = payload.get("_trace") + payload["_trace"] = {**existing, **compact_trace} if isinstance(existing, dict) else compact_trace + return payload + + +def _trace_details( + details: dict[str, Any], + *, + correlation_id: str | None = None, + causation_id: str | None = None, +) -> tuple[dict[str, Any], dict[str, str]]: + active_trace = current_event_trace() + trace: dict[str, str] = {} + clean_correlation_id = normalize_trace_id(correlation_id) + clean_causation_id = normalize_trace_id(causation_id) + if clean_correlation_id or active_trace: + trace["correlation_id"] = clean_correlation_id or active_trace.correlation_id + if clean_causation_id or (active_trace and active_trace.causation_id): + trace["causation_id"] = clean_causation_id or str(active_trace.causation_id) + if not trace: + return details, {} + merged = dict(details) + existing = merged.get("_trace") + merged["_trace"] = {**existing, **trace} if isinstance(existing, dict) else trace + return merged, trace + + +def _restore_trace_after_policy(details: dict[str, Any], trace: dict[str, str]) -> dict[str, Any]: + if not trace: + return details + if details.get("_trace") == trace: + return details + restored = dict(details) + existing = restored.get("_trace") + restored["_trace"] = {**existing, **trace} if isinstance(existing, dict) else trace + return restored + + def audit_event( session: Session, *, @@ -44,6 +146,8 @@ def audit_event( object_type: str | None = None, object_id: str | None = None, details: dict[str, Any] | None = None, + correlation_id: str | None = None, + causation_id: str | None = None, commit: bool = False, ) -> AuditLog: """Persist one audit event. @@ -56,6 +160,16 @@ def audit_event( if scope not in {"tenant", "system"}: raise ValueError(f"Unsupported audit scope: {scope}") + traced_details, trace = _trace_details( + _sanitize_details(details or {}), + correlation_id=correlation_id, + causation_id=causation_id, + ) + stored_details = _restore_trace_after_policy( + sanitize_audit_details_for_policy(session, traced_details), + trace, + ) + item = AuditLog( scope=scope, tenant_id=tenant_id, @@ -64,13 +178,24 @@ def audit_event( action=action, object_type=object_type, object_id=object_id, - details=sanitize_audit_details_for_policy(session, _sanitize_details(details or {})), + details=stored_details, ) session.add(item) + session.flush() + record_change( + session, + module_id=AUDIT_MODULE_ID, + collection=AUDIT_SYSTEM_EVENTS_COLLECTION if scope == "system" else AUDIT_TENANT_EVENTS_COLLECTION, + resource_type="audit_log", + resource_id=item.id, + operation="created", + tenant_id=None if scope == "system" else tenant_id, + actor_type="user" if user_id else ("api_key" if api_key_id else None), + actor_id=user_id or api_key_id, + payload={"scope": scope, "action": action, "object_type": object_type, "object_id": object_id}, + ) if commit: session.commit() - else: - session.flush() return item @@ -83,6 +208,8 @@ def audit_from_principal( object_type: str | None = None, object_id: str | None = None, details: dict[str, Any] | None = None, + correlation_id: str | None = None, + causation_id: str | None = None, commit: bool = False, ) -> AuditLog: return audit_event( @@ -95,5 +222,7 @@ def audit_from_principal( object_type=object_type, object_id=object_id, details=details, + correlation_id=correlation_id, + causation_id=causation_id, commit=commit, ) diff --git a/src/govoplan_core/commands/module_installer.py b/src/govoplan_core/commands/module_installer.py index 7fddd48..35c19b3 100644 --- a/src/govoplan_core/commands/module_installer.py +++ b/src/govoplan_core/commands/module_installer.py @@ -27,6 +27,7 @@ from govoplan_core.core.module_installer import ( update_module_installer_request, update_module_installer_daemon_status, ) +from govoplan_core.core.module_license import issue_module_license, module_license_diagnostics from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog from govoplan_core.core.module_management import ( configured_enabled_modules, @@ -80,10 +81,69 @@ def main() -> int: parser.add_argument("--catalog-signing-key-id", help="Key id to record when signing a package catalog.") parser.add_argument("--catalog-signing-private-key", type=Path, help="PEM Ed25519 private key used by --sign-package-catalog.") parser.add_argument("--catalog-output", type=Path, help="Output path for --sign-package-catalog. Defaults to overwriting the input file.") + parser.add_argument("--validate-license", nargs="?", const="", metavar="PATH", help="Validate a license JSON file, or the configured license when PATH is omitted.") + parser.add_argument("--require-trusted-license", action="store_true", help="Require license validation to trust an Ed25519 signature.") + parser.add_argument("--license-trusted-key", action="append", default=[], metavar="KEY_ID=BASE64_PUBLIC_KEY", help="Trusted Ed25519 license public key; may be repeated.") + parser.add_argument("--license-required-feature", action="append", default=[], help="License feature to check; may be repeated.") + parser.add_argument("--issue-license", type=Path, metavar="PATH", help="Write a signed offline license JSON file.") + parser.add_argument("--license-id", help="License id for --issue-license.") + parser.add_argument("--license-subject", help="License subject/customer for --issue-license.") + parser.add_argument("--license-feature", action="append", default=[], help="Feature entitlement to include in --issue-license; may be repeated.") + parser.add_argument("--license-valid-from", help="License valid_from timestamp. Defaults to now.") + parser.add_argument("--license-valid-until", help="License valid_until timestamp.") + parser.add_argument("--license-signing-key-id", help="Signing key id for --issue-license.") + parser.add_argument("--license-signing-private-key", type=Path, help="PEM Ed25519 private key for --issue-license.") + parser.add_argument("--license-issuer", help="Optional issuer string for --issue-license.") + parser.add_argument("--license-notes", help="Optional operator note for --issue-license.") args = parser.parse_args() runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url) try: + if args.issue_license: + if not args.license_id or not args.license_subject or not args.license_valid_until: + raise ModuleInstallerError("--issue-license requires --license-id, --license-subject, and --license-valid-until.") + if not args.license_signing_key_id or not args.license_signing_private_key: + raise ModuleInstallerError("--issue-license requires --license-signing-key-id and --license-signing-private-key.") + if not [item for item in args.license_feature if item.strip()]: + raise ModuleInstallerError("--issue-license requires at least one --license-feature.") + try: + path = issue_module_license( + path=args.issue_license, + license_id=args.license_id, + subject=args.license_subject, + features=args.license_feature, + valid_from=args.license_valid_from, + valid_until=args.license_valid_until, + signing_key_id=args.license_signing_key_id, + signing_private_key_path=args.license_signing_private_key, + issuer=args.license_issuer, + notes=args.license_notes, + ) + except (OSError, ValueError) as exc: + raise ModuleInstallerError(str(exc)) from exc + _print_result( + { + "issued": True, + "path": str(path), + "license_id": args.license_id, + "subject": args.license_subject, + "features": list(dict.fromkeys(item.strip() for item in args.license_feature if item.strip())), + "valid_until": args.license_valid_until, + "key_id": args.license_signing_key_id, + }, + output_format=args.format, + ) + return 0 + if args.validate_license is not None: + path = Path(args.validate_license).expanduser() if args.validate_license else None + result = module_license_diagnostics( + path, + require_trusted=args.require_trusted_license, + trusted_keys=_trusted_keys_from_args(args.license_trusted_key, flag_name="--license-trusted-key"), + required_features=args.license_required_feature, + ) + _print_result(result, output_format=args.format) + return 0 if result.get("valid") and not result.get("missing_features") else 1 if args.sign_package_catalog: if not args.catalog_signing_key_id or not args.catalog_signing_private_key: raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.") @@ -101,7 +161,7 @@ def main() -> int: path, require_trusted=args.require_signed_catalog, approved_channels=tuple(args.approved_catalog_channel), - trusted_keys=_trusted_catalog_keys_from_args(args.catalog_trusted_key), + trusted_keys=_trusted_keys_from_args(args.catalog_trusted_key, flag_name="--catalog-trusted-key"), ) _print_result(result, output_format=args.format) return 0 if result.get("valid") else 1 @@ -322,6 +382,7 @@ def _process_request(*, request: dict[str, object], args: argparse.Namespace, ru health_urls=_list_option(options, "health_urls") or _list_option(options, "health_url") or ([args.health_url] if args.health_url else []), health_timeout_seconds=_float_option(options, "health_timeout_seconds", args.health_timeout_seconds), health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds), + request_context=_request_context(request), ) update_module_installer_request( runtime_dir=runtime_dir, @@ -362,6 +423,18 @@ def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]: } +def _request_context(request: dict[str, object]) -> dict[str, object]: + context: dict[str, object] = { + "request_id": request.get("request_id"), + "requested_by": request.get("requested_by"), + } + if request.get("retry_of"): + context["retry_of"] = request["retry_of"] + if isinstance(request.get("trace"), dict): + context["trace"] = request["trace"] + return {key: value for key, value in context.items() if value is not None} + + def _str_option(options: object, key: str) -> str | None: if not isinstance(options, dict): return None @@ -403,14 +476,14 @@ def _list_option(options: object, key: str) -> list[str]: return [] -def _trusted_catalog_keys_from_args(values: list[str]) -> dict[str, str] | None: +def _trusted_keys_from_args(values: list[str], *, flag_name: str) -> dict[str, str] | None: if not values: return None keys: dict[str, str] = {} for value in values: key_id, separator, public_key = value.partition("=") if not separator or not key_id.strip() or not public_key.strip(): - raise ModuleInstallerError("--catalog-trusted-key must use KEY_ID=BASE64_PUBLIC_KEY.") + raise ModuleInstallerError(f"{flag_name} must use KEY_ID=BASE64_PUBLIC_KEY.") keys[key_id.strip()] = public_key.strip() return keys diff --git a/src/govoplan_core/core/access.py b/src/govoplan_core/core/access.py index df7fe5e..d21dd6b 100644 --- a/src/govoplan_core/core/access.py +++ b/src/govoplan_core/core/access.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime -from typing import Literal, Protocol, runtime_checkable +from typing import Literal, Protocol, cast, runtime_checkable from govoplan_core.core.modules import AccessDecision @@ -17,11 +17,12 @@ CAPABILITY_ACCESS_PRINCIPAL_RESOLVER = "access.principalResolver" CAPABILITY_ACCESS_DIRECTORY = "access.directory" CAPABILITY_ACCESS_PERMISSION_EVALUATOR = "access.permissionEvaluator" CAPABILITY_ACCESS_RESOURCE_ACCESS = "access.resourceAccess" +CAPABILITY_ACCESS_SEMANTIC_DIRECTORY = "access.semanticDirectory" +CAPABILITY_ACCESS_EXPLANATION = "access.explanation" CAPABILITY_ACCESS_TENANT_PROVISIONER = "access.tenantProvisioner" CAPABILITY_ACCESS_ADMINISTRATION = "access.administration" CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER = "access.governanceMaterializer" CAPABILITY_TENANCY_TENANT_RESOLVER = "tenancy.tenantResolver" -CAPABILITY_SECURITY_SECRET_PROVIDER = "security.secretProvider" CAPABILITY_AUDIT_SINK = "audit.sink" ACCESS_CAPABILITY_NAMES = frozenset( @@ -30,20 +31,47 @@ ACCESS_CAPABILITY_NAMES = frozenset( CAPABILITY_ACCESS_DIRECTORY, CAPABILITY_ACCESS_PERMISSION_EVALUATOR, CAPABILITY_ACCESS_RESOURCE_ACCESS, + CAPABILITY_ACCESS_SEMANTIC_DIRECTORY, + CAPABILITY_ACCESS_EXPLANATION, CAPABILITY_ACCESS_TENANT_PROVISIONER, CAPABILITY_ACCESS_ADMINISTRATION, CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER, CAPABILITY_TENANCY_TENANT_RESOLVER, - CAPABILITY_SECURITY_SECRET_PROVIDER, CAPABILITY_AUDIT_SINK, } ) AuthMethod = Literal["session", "api_key", "service_account"] -AccessSubjectKind = Literal["account", "user", "group", "role", "tenant", "api_key", "service_account"] +AccessSubjectKind = Literal[ + "identity", + "account", + "user", + "group", + "role", + "tenant", + "organization_unit", + "function", + "api_key", + "service_account", +] AccessStatus = Literal["active", "inactive", "suspended"] PermissionLevel = Literal["system", "tenant"] AuditOutcome = Literal["success", "failure", "denied", "unknown"] +FunctionAssignmentSource = Literal["direct", "delegated", "acting_for", "directory", "governance", "system"] +FunctionDelegationMode = Literal["delegate", "act_in_place"] +AccessProvenanceKind = Literal[ + "identity", + "account", + "tenant_membership", + "organization_unit", + "function", + "group", + "role", + "right", + "delegation", + "policy", + "system_actor", +] @dataclass(frozen=True, slots=True) @@ -51,15 +79,75 @@ class PrincipalRef: account_id: str membership_id: str | None tenant_id: str | None + identity_id: str | None = None scopes: frozenset[str] = field(default_factory=frozenset) group_ids: frozenset[str] = field(default_factory=frozenset) + role_ids: frozenset[str] = field(default_factory=frozenset) + function_assignment_ids: frozenset[str] = field(default_factory=frozenset) + delegation_ids: frozenset[str] = field(default_factory=frozenset) auth_method: AuthMethod = "session" api_key_id: str | None = None session_id: str | None = None service_account_id: str | None = None + acting_for_account_id: str | None = None email: str | None = None display_name: str | None = None + def to_dict(self) -> dict[str, object]: + return { + "account_id": self.account_id, + "membership_id": self.membership_id, + "tenant_id": self.tenant_id, + "identity_id": self.identity_id, + "scopes": sorted(self.scopes), + "group_ids": sorted(self.group_ids), + "role_ids": sorted(self.role_ids), + "function_assignment_ids": sorted(self.function_assignment_ids), + "delegation_ids": sorted(self.delegation_ids), + "auth_method": self.auth_method, + "api_key_id": self.api_key_id, + "session_id": self.session_id, + "service_account_id": self.service_account_id, + "acting_for_account_id": self.acting_for_account_id, + "email": self.email, + "display_name": self.display_name, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> PrincipalRef: + return cls( + account_id=str(value["account_id"]), + membership_id=_optional_str(value.get("membership_id")), + tenant_id=_optional_str(value.get("tenant_id")), + identity_id=_optional_str(value.get("identity_id")), + scopes=_string_frozenset(value.get("scopes")), + group_ids=_string_frozenset(value.get("group_ids")), + role_ids=_string_frozenset(value.get("role_ids")), + function_assignment_ids=_string_frozenset(value.get("function_assignment_ids")), + delegation_ids=_string_frozenset(value.get("delegation_ids")), + auth_method=cast(AuthMethod, str(value.get("auth_method") or "session")), + api_key_id=_optional_str(value.get("api_key_id")), + session_id=_optional_str(value.get("session_id")), + service_account_id=_optional_str(value.get("service_account_id")), + acting_for_account_id=_optional_str(value.get("acting_for_account_id")), + email=_optional_str(value.get("email")), + display_name=_optional_str(value.get("display_name")), + ) + + +def _optional_str(value: object | None) -> str | None: + return str(value) if value is not None else None + + +def _string_frozenset(value: object | None) -> frozenset[str]: + if value is None: + return frozenset() + if isinstance(value, str): + return frozenset({value}) + if isinstance(value, Iterable): + return frozenset(str(item) for item in value) + raise TypeError("PrincipalRef collection fields must be strings or iterables") + @dataclass(frozen=True, slots=True) class AccountRef: @@ -87,6 +175,15 @@ class UserRef: status: AccessStatus = "active" +@dataclass(frozen=True, slots=True) +class IdentityRef: + id: str + display_name: str | None = None + primary_account_id: str | None = None + account_ids: tuple[str, ...] = () + status: AccessStatus = "active" + + @dataclass(frozen=True, slots=True) class GroupRef: id: str @@ -105,6 +202,79 @@ class RoleRef: protected: bool = False +@dataclass(frozen=True, slots=True) +class OrganizationUnitRef: + id: str + tenant_id: str + name: str + parent_id: str | None = None + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class FunctionRef: + id: str + tenant_id: str + organization_unit_id: str + slug: str + name: str + role_ids: tuple[str, ...] = () + delegable: bool = False + act_in_place_allowed: bool = False + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class FunctionAssignmentRef: + id: str + tenant_id: str + account_id: str + function_id: str + organization_unit_id: str + identity_id: str | None = None + applies_to_subunits: bool = False + source: FunctionAssignmentSource = "direct" + delegated_from_assignment_id: str | None = None + acting_for_account_id: str | None = None + valid_from: datetime | None = None + valid_until: datetime | None = None + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class FunctionDelegationRef: + id: str + tenant_id: str + function_assignment_id: str + delegator_account_id: str + delegate_account_id: str + mode: FunctionDelegationMode = "delegate" + reason: str | None = None + valid_from: datetime | None = None + valid_until: datetime | None = None + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class AccessDecisionProvenance: + kind: AccessProvenanceKind + id: str | None = None + label: str | None = None + tenant_id: str | None = None + source: str | None = None + details: Mapping[str, object] = field(default_factory=dict) + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "id": self.id, + "label": self.label, + "tenant_id": self.tenant_id, + "source": self.source, + "details": dict(self.details), + } + + @dataclass(frozen=True, slots=True) class AccessSubjectRef: kind: AccessSubjectKind @@ -143,6 +313,8 @@ class AuditEvent: resource_type: str | None = None resource_id: str | None = None occurred_at: datetime | None = None + correlation_id: str | None = None + causation_id: str | None = None details: Mapping[str, object] = field(default_factory=dict) @@ -191,6 +363,43 @@ class AccessDirectory(Protocol): ... +@runtime_checkable +class AccessSemanticDirectory(AccessDirectory, Protocol): + def get_identity(self, identity_id: str) -> IdentityRef | None: + ... + + def accounts_for_identity(self, identity_id: str) -> Sequence[AccountRef]: + ... + + def get_organization_unit(self, organization_unit_id: str) -> OrganizationUnitRef | None: + ... + + def organization_units_for_tenant(self, tenant_id: str) -> Sequence[OrganizationUnitRef]: + ... + + def get_function(self, function_id: str) -> FunctionRef | None: + ... + + def functions_for_organization_unit( + self, + organization_unit_id: str, + *, + include_subunits: bool = False, + ) -> Sequence[FunctionRef]: + ... + + def get_function_assignment(self, assignment_id: str) -> FunctionAssignmentRef | None: + ... + + def function_assignments_for_account( + self, + account_id: str, + *, + tenant_id: str | None = None, + ) -> Sequence[FunctionAssignmentRef]: + ... + + @runtime_checkable class PermissionEvaluator(Protocol): def scopes_grant(self, scopes: Iterable[str], required_scope: str) -> bool: @@ -209,6 +418,26 @@ class ResourceAccessService(Protocol): ... +@runtime_checkable +class AccessExplanationService(Protocol): + def explain_scope_provenance( + self, + principal: PrincipalRef, + required_scope: str, + ) -> Sequence[AccessDecisionProvenance]: + ... + + def explain_resource_provenance( + self, + principal: PrincipalRef, + *, + resource_type: str, + resource_id: str, + action: str, + ) -> Sequence[AccessDecisionProvenance]: + ... + + @runtime_checkable class TenantAccessProvisioner(Protocol): def ensure_default_roles(self, session: object, tenant: object | None = None) -> Mapping[str, object]: @@ -254,18 +483,6 @@ class AccessGovernanceMaterializer(Protocol): ... -@runtime_checkable -class SecretProvider(Protocol): - def store_secret(self, *, scope: str, name: str, value: str) -> str: - ... - - def read_secret(self, secret_ref: str) -> str | None: - ... - - def delete_secret(self, secret_ref: str) -> None: - ... - - @runtime_checkable class AuditSink(Protocol): def record(self, event: AuditEvent) -> None: diff --git a/src/govoplan_core/core/change_sequence.py b/src/govoplan_core/core/change_sequence.py new file mode 100644 index 0000000..e499b37 --- /dev/null +++ b/src/govoplan_core/core/change_sequence.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Iterable, Sequence + +from sqlalchemy import BigInteger, DateTime, Index, Integer, JSON, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, Session, mapped_column + +from govoplan_core.db.base import Base, utcnow + +WATERMARK_PREFIX = "seq:" + + +class ChangeSequenceEntry(Base): + __tablename__ = "core_change_sequence" + __table_args__ = ( + Index("ix_core_change_sequence_tenant_module_id", "tenant_id", "module_id", "id"), + Index("ix_core_change_sequence_collection_id", "collection", "id"), + Index("ix_core_change_sequence_resource", "module_id", "resource_type", "resource_id"), + ) + + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + tenant_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True) + module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + collection: Mapped[str] = mapped_column(String(150), nullable=False, index=True) + resource_type: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + resource_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + operation: Mapped[str] = mapped_column(String(30), nullable=False, index=True) + actor_type: Mapped[str | None] = mapped_column(String(30), nullable=True, index=True) + actor_id: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True) + payload: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False, index=True) + + +class ChangeSequenceRetentionFloor(Base): + __tablename__ = "core_change_sequence_retention_floor" + __table_args__ = ( + UniqueConstraint("tenant_key", "module_id", "collection", name="uq_core_change_sequence_retention_scope"), + Index("ix_core_change_sequence_retention_scope", "tenant_key", "module_id", "collection"), + ) + + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + tenant_key: Mapped[str] = mapped_column(String(36), nullable=False, default="") + module_id: Mapped[str] = mapped_column(String(100), nullable=False) + collection: Mapped[str] = mapped_column(String(150), nullable=False) + min_valid_sequence: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), nullable=False, default=0) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False) + + +def _tenant_key(tenant_id: str | None) -> str: + return tenant_id or "" + + +def encode_sequence_watermark(sequence_id: int | None) -> str: + return f"{WATERMARK_PREFIX}{int(sequence_id or 0)}" + + +def decode_sequence_watermark(value: str | None) -> int: + if value is None or not value.strip(): + return 0 + raw = value.strip() + if raw.startswith(WATERMARK_PREFIX): + raw = raw[len(WATERMARK_PREFIX):] + try: + sequence_id = int(raw) + except ValueError as exc: + raise ValueError("Invalid delta watermark") from exc + if sequence_id < 0: + raise ValueError("Invalid delta watermark") + return sequence_id + + +def record_change( + session: Session, + *, + module_id: str, + collection: str, + resource_type: str, + resource_id: str, + operation: str, + tenant_id: str | None = None, + actor_type: str | None = None, + actor_id: str | None = None, + payload: dict[str, Any] | None = None, +) -> ChangeSequenceEntry: + entry = ChangeSequenceEntry( + tenant_id=tenant_id, + module_id=module_id, + collection=collection, + resource_type=resource_type, + resource_id=resource_id, + operation=operation, + actor_type=actor_type, + actor_id=actor_id, + payload=payload or {}, + ) + session.add(entry) + return entry + + +def max_sequence_id( + session: Session, + *, + tenant_id: str | None = None, + module_id: str | None = None, + collections: Sequence[str] | None = None, +) -> int: + query = session.query(func.max(ChangeSequenceEntry.id)) + if tenant_id is not None: + query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id) + if module_id is not None: + query = query.filter(ChangeSequenceEntry.module_id == module_id) + if collections: + query = query.filter(ChangeSequenceEntry.collection.in_(tuple(collections))) + return int(query.scalar() or 0) + + +def min_sequence_id( + session: Session, + *, + tenant_id: str | None = None, + module_id: str | None = None, + collections: Sequence[str] | None = None, +) -> int: + query = session.query(func.min(ChangeSequenceEntry.id)) + if tenant_id is not None: + query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id) + if module_id is not None: + query = query.filter(ChangeSequenceEntry.module_id == module_id) + if collections: + query = query.filter(ChangeSequenceEntry.collection.in_(tuple(collections))) + return int(query.scalar() or 0) + + +def sequence_watermark_is_expired( + session: Session, + *, + since: int, + tenant_id: str | None = None, + module_id: str | None = None, + collections: Sequence[str] | None = None, +) -> bool: + floor = retained_sequence_floor( + session, + tenant_id=tenant_id, + module_id=module_id, + collections=collections, + ) + return floor > 0 and since < floor + + +def retained_sequence_floor( + session: Session, + *, + tenant_id: str | None = None, + module_id: str | None = None, + collections: Sequence[str] | None = None, +) -> int: + query = session.query(func.max(ChangeSequenceRetentionFloor.min_valid_sequence)) + query = query.filter(ChangeSequenceRetentionFloor.tenant_key == _tenant_key(tenant_id)) + if module_id is not None: + query = query.filter(ChangeSequenceRetentionFloor.module_id == module_id) + if collections: + query = query.filter(ChangeSequenceRetentionFloor.collection.in_(tuple(collections))) + return int(query.scalar() or 0) + + +def prune_sequence_entries( + session: Session, + *, + before_or_equal_id: int, + tenant_id: str | None = None, + module_id: str | None = None, + collections: Sequence[str] | None = None, +) -> int: + """Delete old sequence rows and record the new minimum safe watermark. + + After pruning rows up to `before_or_equal_id`, clients with watermarks below + that value cannot be replayed safely and should receive a full snapshot. + """ + + cutoff = int(before_or_equal_id) + if cutoff <= 0: + return 0 + query = session.query(ChangeSequenceEntry).filter(ChangeSequenceEntry.id <= cutoff) + if tenant_id is not None: + query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id) + if module_id is not None: + query = query.filter(ChangeSequenceEntry.module_id == module_id) + collection_values = tuple(collections or ()) + if collection_values: + query = query.filter(ChangeSequenceEntry.collection.in_(collection_values)) + scopes = { + (_tenant_key(entry.tenant_id), entry.module_id, entry.collection) + for entry in query.with_entities( + ChangeSequenceEntry.tenant_id, + ChangeSequenceEntry.module_id, + ChangeSequenceEntry.collection, + ).distinct() + } + deleted = query.delete(synchronize_session=False) + for tenant_key, scoped_module_id, collection in scopes: + floor = session.query(ChangeSequenceRetentionFloor).filter( + ChangeSequenceRetentionFloor.tenant_key == tenant_key, + ChangeSequenceRetentionFloor.module_id == scoped_module_id, + ChangeSequenceRetentionFloor.collection == collection, + ).one_or_none() + if floor is None: + floor = ChangeSequenceRetentionFloor( + tenant_key=tenant_key, + module_id=scoped_module_id, + collection=collection, + min_valid_sequence=cutoff, + ) + else: + floor.min_valid_sequence = max(int(floor.min_valid_sequence or 0), cutoff) + session.add(floor) + return int(deleted) + + +def sequence_entries_since( + session: Session, + *, + since: int, + tenant_id: str | None = None, + module_id: str | None = None, + collections: Iterable[str] | None = None, + limit: int = 500, +) -> list[ChangeSequenceEntry]: + query = session.query(ChangeSequenceEntry).filter(ChangeSequenceEntry.id > since) + if tenant_id is not None: + query = query.filter(ChangeSequenceEntry.tenant_id == tenant_id) + if module_id is not None: + query = query.filter(ChangeSequenceEntry.module_id == module_id) + collection_values = tuple(collections or ()) + if collection_values: + query = query.filter(ChangeSequenceEntry.collection.in_(collection_values)) + return query.order_by(ChangeSequenceEntry.id.asc()).limit(max(1, limit)).all() + + +__all__ = [ + "ChangeSequenceEntry", + "ChangeSequenceRetentionFloor", + "decode_sequence_watermark", + "encode_sequence_watermark", + "max_sequence_id", + "min_sequence_id", + "prune_sequence_entries", + "record_change", + "retained_sequence_floor", + "sequence_entries_since", + "sequence_watermark_is_expired", +] diff --git a/src/govoplan_core/core/configuration_control.py b/src/govoplan_core/core/configuration_control.py new file mode 100644 index 0000000..d1acc04 --- /dev/null +++ b/src/govoplan_core/core/configuration_control.py @@ -0,0 +1,418 @@ +from __future__ import annotations + +import hashlib +import json +import uuid +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.orm import Session + +from govoplan_core.admin.settings import get_system_settings +from govoplan_core.core.change_sequence import record_change +from govoplan_core.core.configuration_safety import classify_configuration_field, plan_configuration_change +from govoplan_core.core.maintenance import saved_maintenance_mode +from govoplan_core.security.time import utc_now + + +CONFIGURATION_CONTROL_SETTINGS_KEY = "_configuration_control" +CONFIGURATION_CONTROL_MODULE_ID = "core" +CONFIGURATION_CHANGES_COLLECTION = "core.configuration_changes" +CONFIGURATION_CHANGE_REQUEST_RESOURCE = "configuration_change_request" +CONFIGURATION_CHANGE_RECORD_RESOURCE = "configuration_change_record" +MAX_CONFIGURATION_CHANGE_HISTORY = 200 +MAX_CONFIGURATION_CHANGE_REQUESTS = 200 + + +class ConfigurationControlError(Exception): + def __init__(self, message: str, *, code: str = "configuration_change_blocked", plan: dict[str, Any] | None = None) -> None: + super().__init__(message) + self.code = code + self.plan = plan or {} + + +@dataclass(frozen=True, slots=True) +class ConfigurationChangeApproval: + request: dict[str, Any] | None + plan: dict[str, Any] + + +def configuration_value_digest(value: object) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def configuration_control_snapshot(session: Session) -> dict[str, list[dict[str, Any]]]: + item = get_system_settings(session) + control = _control_payload(item.settings or {}) + return { + "requests": list(control.get("requests", [])), + "history": list(control.get("history", [])), + } + + +def create_configuration_change_request( + session: Session, + *, + key: str, + value: object, + actor_user_id: str, + actor_scopes: tuple[str, ...] | list[str], + dry_run: bool, + target: dict[str, Any] | None = None, + reason: str | None = None, +) -> dict[str, Any]: + field = classify_configuration_field(key) + if field is None: + raise ConfigurationControlError( + "This setting is not in the configuration safety catalog.", + code="unknown_configuration_field", + plan=plan_configuration_change(key, actor_scopes=actor_scopes, value=value).to_dict(), + ) + if not field.ui_managed: + raise ConfigurationControlError( + "This setting is deployment-managed and cannot be edited through the UI.", + code="deployment_managed", + plan=plan_configuration_change(key, actor_scopes=actor_scopes, value=value).to_dict(), + ) + + item = get_system_settings(session) + settings = dict(item.settings or {}) + control = _control_payload(settings) + now = _now() + request = { + "id": f"cfgreq-{uuid.uuid4().hex}", + "key": field.key, + "label": field.label, + "target": dict(target or {}), + "value_hash": configuration_value_digest(value), + "value_preview": _sanitize_value(field.key, value), + "dry_run": bool(dry_run), + "requested_by": actor_user_id, + "requested_at": now, + "updated_at": now, + "reason": reason.strip() if isinstance(reason, str) and reason.strip() else None, + "status": "pending_approval" if field.two_person_approval_required else "approved", + "approvals": [ + { + "user_id": actor_user_id, + "approved_at": now, + "role": "requester", + } + ], + "plan": plan_configuration_change( + field.key, + actor_scopes=actor_scopes, + value=value, + dry_run=bool(dry_run), + maintenance_mode=saved_maintenance_mode(session).enabled, + approval_count=1, + include_env_only=False, + ).to_dict(), + } + control["requests"] = _trim_requests([request, *list(control.get("requests", []))]) + settings[CONFIGURATION_CONTROL_SETTINGS_KEY] = control + item.settings = settings + session.add(item) + session.flush() + _record_configuration_control_change( + session, + resource_type=CONFIGURATION_CHANGE_REQUEST_RESOURCE, + resource_id=str(request["id"]), + operation="created", + actor_user_id=actor_user_id, + payload={"key": field.key, "status": request["status"]}, + ) + return dict(request) + + +def approve_configuration_change_request( + session: Session, + *, + request_id: str, + actor_user_id: str, + actor_scopes: tuple[str, ...] | list[str], + reason: str | None = None, +) -> dict[str, Any]: + item = get_system_settings(session) + settings = dict(item.settings or {}) + control = _control_payload(settings) + requests = list(control.get("requests", [])) + index, request = _find_request(requests, request_id) + if request is None: + raise ConfigurationControlError("Configuration change request not found.", code="configuration_request_not_found") + if request.get("status") in {"applied", "rejected"}: + raise ConfigurationControlError("Configuration change request is no longer approvable.", code="configuration_request_closed") + if request.get("requested_by") == actor_user_id: + raise ConfigurationControlError("The requester cannot be the second approver.", code="second_approver_required") + approvals = list(request.get("approvals", [])) + if not any(item.get("user_id") == actor_user_id for item in approvals if isinstance(item, dict)): + approvals.append( + { + "user_id": actor_user_id, + "approved_at": _now(), + "role": "approver", + "reason": reason.strip() if isinstance(reason, str) and reason.strip() else None, + } + ) + request = dict(request) + request["approvals"] = approvals + request["updated_at"] = _now() + request["status"] = "approved" if _approval_count(request) >= 2 else "pending_approval" + request["plan"] = plan_configuration_change( + str(request.get("key") or ""), + actor_scopes=actor_scopes, + value=request.get("value_preview"), + dry_run=bool(request.get("dry_run")), + maintenance_mode=saved_maintenance_mode(session).enabled, + approval_count=_approval_count(request), + include_env_only=False, + ).to_dict() + requests[index] = request + control["requests"] = _trim_requests(requests) + settings[CONFIGURATION_CONTROL_SETTINGS_KEY] = control + item.settings = settings + session.add(item) + session.flush() + _record_configuration_control_change( + session, + resource_type=CONFIGURATION_CHANGE_REQUEST_RESOURCE, + resource_id=str(request["id"]), + operation="updated", + actor_user_id=actor_user_id, + payload={"key": request.get("key"), "status": request.get("status")}, + ) + return dict(request) + + +def ensure_configuration_change_allowed( + session: Session, + *, + key: str, + value: object, + actor_user_id: str, + actor_scopes: tuple[str, ...] | list[str], + change_request_id: str | None = None, + target: dict[str, Any] | None = None, + dry_run: bool = False, +) -> ConfigurationChangeApproval: + field = classify_configuration_field(key) + if field is None: + plan = plan_configuration_change(key, actor_scopes=actor_scopes, value=value).to_dict() + raise ConfigurationControlError("This setting is not in the configuration safety catalog.", code="unknown_configuration_field", plan=plan) + + request: dict[str, Any] | None = None + approval_count = 0 + dry_run_satisfied = dry_run + requires_control = field.ui_managed and (field.dry_run_required or field.two_person_approval_required or field.maintenance_required) + if change_request_id: + request = _request_by_id(session, change_request_id) + if request is None: + raise ConfigurationControlError("Configuration change request not found.", code="configuration_request_not_found") + if request.get("status") in {"applied", "rejected"}: + raise ConfigurationControlError("Configuration change request is closed.", code="configuration_request_closed") + if request.get("key") != field.key: + raise ConfigurationControlError("Configuration change request does not match this setting.", code="configuration_request_mismatch") + if request.get("value_hash") != configuration_value_digest(value): + raise ConfigurationControlError("Configuration change request value does not match this apply payload.", code="configuration_value_mismatch") + approval_count = _approval_count(request) + dry_run_satisfied = bool(request.get("dry_run")) or dry_run_satisfied + elif requires_control: + plan = plan_configuration_change( + field.key, + actor_scopes=actor_scopes, + value=value, + dry_run=dry_run_satisfied, + maintenance_mode=saved_maintenance_mode(session).enabled, + approval_count=approval_count, + include_env_only=False, + ).to_dict() + raise ConfigurationControlError( + "This configuration change requires a recorded change request.", + code="configuration_request_required", + plan=plan, + ) + + plan = plan_configuration_change( + field.key, + actor_scopes=actor_scopes, + value=value, + dry_run=dry_run_satisfied, + maintenance_mode=saved_maintenance_mode(session).enabled, + approval_count=approval_count, + include_env_only=False, + ).to_dict() + if plan.get("blockers"): + raise ConfigurationControlError("Configuration change is blocked by safety guardrails.", plan=plan) + return ConfigurationChangeApproval(request=request, plan=plan) + + +def record_configuration_change_applied( + session: Session, + *, + key: str, + before_value: object, + after_value: object, + actor_user_id: str, + approval: ConfigurationChangeApproval | None = None, + target: dict[str, Any] | None = None, + audit_event: str | None = None, +) -> dict[str, Any]: + item = get_system_settings(session) + settings = dict(item.settings or {}) + control = _control_payload(settings) + sequence = int(control.get("sequence") or 0) + 1 + request = approval.request if approval else None + record = { + "id": f"cfgchg-{sequence:06d}-{uuid.uuid4().hex[:8]}", + "version": sequence, + "key": key, + "target": dict(target or {}), + "actor_user_id": actor_user_id, + "approval_request_id": request.get("id") if request else None, + "approval_user_ids": [item.get("user_id") for item in request.get("approvals", []) if isinstance(item, dict)] if request else [], + "audit_event": audit_event, + "before": _sanitize_value(key, before_value), + "after": _sanitize_value(key, after_value), + "rollback_value": _sanitize_value(key, before_value), + "plan": approval.plan if approval else plan_configuration_change(key, actor_scopes=(), value=after_value).to_dict(), + "created_at": _now(), + "status": "applied", + } + control["sequence"] = sequence + control["history"] = _trim_history([record, *list(control.get("history", []))]) + if request: + requests = list(control.get("requests", [])) + index, stored = _find_request(requests, str(request.get("id"))) + if stored is not None: + stored = dict(stored) + stored["status"] = "applied" + stored["applied_change_id"] = record["id"] + stored["updated_at"] = _now() + requests[index] = stored + control["requests"] = _trim_requests(requests) + settings[CONFIGURATION_CONTROL_SETTINGS_KEY] = control + item.settings = settings + session.add(item) + session.flush() + _record_configuration_control_change( + session, + resource_type=CONFIGURATION_CHANGE_RECORD_RESOURCE, + resource_id=str(record["id"]), + operation="created", + actor_user_id=actor_user_id, + payload={"key": key, "status": record["status"], "version": record["version"]}, + ) + if request: + _record_configuration_control_change( + session, + resource_type=CONFIGURATION_CHANGE_REQUEST_RESOURCE, + resource_id=str(request.get("id")), + operation="updated", + actor_user_id=actor_user_id, + payload={"key": key, "status": "applied", "applied_change_id": record["id"]}, + ) + return dict(record) + + +def _record_configuration_control_change( + session: Session, + *, + resource_type: str, + resource_id: str, + operation: str, + actor_user_id: str, + payload: dict[str, Any], +) -> None: + record_change( + session, + module_id=CONFIGURATION_CONTROL_MODULE_ID, + collection=CONFIGURATION_CHANGES_COLLECTION, + resource_type=resource_type, + resource_id=resource_id, + operation=operation, + actor_type="user", + actor_id=actor_user_id, + payload=payload, + ) + + +def _request_by_id(session: Session, request_id: str) -> dict[str, Any] | None: + item = get_system_settings(session) + _index, request = _find_request(list(_control_payload(item.settings or {}).get("requests", [])), request_id) + return dict(request) if request is not None else None + + +def _find_request(requests: list[dict[str, Any]], request_id: str) -> tuple[int, dict[str, Any] | None]: + for index, request in enumerate(requests): + if request.get("id") == request_id: + return index, request + return -1, None + + +def _control_payload(settings: dict[str, Any]) -> dict[str, Any]: + raw = settings.get(CONFIGURATION_CONTROL_SETTINGS_KEY) + if not isinstance(raw, dict): + return {"sequence": 0, "requests": [], "history": []} + return { + "sequence": int(raw.get("sequence") or 0), + "requests": [dict(item) for item in raw.get("requests", []) if isinstance(item, dict)], + "history": [dict(item) for item in raw.get("history", []) if isinstance(item, dict)], + } + + +def _approval_count(request: dict[str, Any]) -> int: + user_ids = [] + for item in request.get("approvals", []): + if isinstance(item, dict) and item.get("user_id"): + user_ids.append(str(item["user_id"])) + return len(set(user_ids)) + + +def _sanitize_value(key: str, value: object) -> object: + field = classify_configuration_field(key) + if field is not None and field.secret_handling in {"reference_only", "env_only"}: + return _redact_secrets(value) + return _redact_secrets(value) + + +def _redact_secrets(value: object) -> object: + if isinstance(value, dict): + result: dict[str, object] = {} + for key, item in value.items(): + if str(key).strip().casefold() in {"password", "token", "secret", "api_key", "access_key", "secret_key"}: + result[str(key)] = "" + else: + result[str(key)] = _redact_secrets(item) + return result + if isinstance(value, list): + return [_redact_secrets(item) for item in value] + return value + + +def _trim_requests(requests: list[dict[str, Any]]) -> list[dict[str, Any]]: + return requests[:MAX_CONFIGURATION_CHANGE_REQUESTS] + + +def _trim_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]: + return history[:MAX_CONFIGURATION_CHANGE_HISTORY] + + +def _now() -> str: + return utc_now().isoformat() + + +__all__ = [ + "ConfigurationChangeApproval", + "ConfigurationControlError", + "CONFIGURATION_CHANGE_RECORD_RESOURCE", + "CONFIGURATION_CHANGE_REQUEST_RESOURCE", + "CONFIGURATION_CHANGES_COLLECTION", + "CONFIGURATION_CONTROL_MODULE_ID", + "approve_configuration_change_request", + "configuration_control_snapshot", + "configuration_value_digest", + "create_configuration_change_request", + "ensure_configuration_change_allowed", + "record_configuration_change_applied", +] diff --git a/src/govoplan_core/core/configuration_packages.py b/src/govoplan_core/core/configuration_packages.py new file mode 100644 index 0000000..c9769b1 --- /dev/null +++ b/src/govoplan_core/core/configuration_packages.py @@ -0,0 +1,883 @@ +from __future__ import annotations + +import base64 +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +import json +import os +from typing import Any, Literal, Protocol, runtime_checkable +import urllib.error +import urllib.request + +from govoplan_core.core.module_package_catalog import ( + _canonical_catalog_bytes, + _catalog_freshness_state, + _catalog_optional_text, + _catalog_sequence, + _catalog_signature_state, + _catalog_source_exists, + _catalog_source_type, + _is_http_url, + _load_private_key, + _parse_trusted_keys, +) + + +CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider" + +DiagnosticSeverity = Literal["blocker", "warning", "info"] +PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"] + + +@dataclass(frozen=True, slots=True) +class ConfigurationModuleRequirement: + module_id: str + version: str | None = None + optional: bool = False + + def to_dict(self) -> dict[str, object]: + return {"module_id": self.module_id, "version": self.version, "optional": self.optional} + + +@dataclass(frozen=True, slots=True) +class ConfigurationPackageFragment: + module_id: str + fragment_type: str + fragment_id: str | None = None + payload: Mapping[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, object]: + return { + "module_id": self.module_id, + "fragment_type": self.fragment_type, + "fragment_id": self.fragment_id, + "payload": dict(self.payload), + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationPackageManifest: + package_id: str + name: str + version: str + description: str | None = None + publisher: str | None = None + category: str | None = None + license: str | None = None + required_modules: tuple[ConfigurationModuleRequirement, ...] = () + required_capabilities: tuple[str, ...] = () + optional_modules: tuple[ConfigurationModuleRequirement, ...] = () + fragments: tuple[ConfigurationPackageFragment, ...] = () + data_requirements: tuple[dict[str, object], ...] = () + tags: tuple[str, ...] = () + artifact_ref: str | None = None + artifact_sha256: str | None = None + signature: Mapping[str, Any] | None = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageManifest": + return cls( + package_id=_required_str(value, "package_id"), + name=_required_str(value, "name"), + version=_required_str(value, "version"), + description=_optional_str(value, "description"), + publisher=_optional_str(value, "publisher"), + category=_optional_str(value, "category"), + license=_optional_str(value, "license"), + required_modules=tuple(_module_requirements(value.get("required_modules"), optional=False)), + required_capabilities=tuple(_string_list(value.get("required_capabilities"))), + optional_modules=tuple(_module_requirements(value.get("optional_modules"), optional=True)), + fragments=tuple(_fragments(value.get("fragments"))), + data_requirements=tuple(_object_list(value.get("data_requirements"), field_name="data_requirements")), + tags=tuple(_string_list(value.get("tags"))), + artifact_ref=_optional_str(value, "artifact_ref"), + artifact_sha256=_optional_str(value, "artifact_sha256"), + signature=value.get("signature") if isinstance(value.get("signature"), Mapping) else None, + ) + + def to_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "package_id": self.package_id, + "name": self.name, + "version": self.version, + "required_modules": [item.to_dict() for item in self.required_modules], + "required_capabilities": list(self.required_capabilities), + "optional_modules": [item.to_dict() for item in self.optional_modules], + "fragments": [item.to_dict() for item in self.fragments], + "data_requirements": [dict(item) for item in self.data_requirements], + "tags": list(self.tags), + } + for key, value in ( + ("description", self.description), + ("publisher", self.publisher), + ("category", self.category), + ("license", self.license), + ("artifact_ref", self.artifact_ref), + ("artifact_sha256", self.artifact_sha256), + ): + if value is not None: + payload[key] = value + if self.signature is not None: + payload["signature"] = dict(self.signature) + return payload + + +@dataclass(frozen=True, slots=True) +class ConfigurationDiagnostic: + severity: DiagnosticSeverity + code: str + message: str + module_id: str | None = None + object_ref: str | None = None + resolution: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "severity": self.severity, + "code": self.code, + "message": self.message, + "module_id": self.module_id, + "object_ref": self.object_ref, + "resolution": self.resolution, + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationPlanItem: + action: PlanAction + module_id: str + fragment_type: str + fragment_id: str | None = None + summary: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "action": self.action, + "module_id": self.module_id, + "fragment_type": self.fragment_type, + "fragment_id": self.fragment_id, + "summary": self.summary, + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationRequiredData: + key: str + label: str + data_type: str = "string" + required: bool = True + secret: bool = False + description: str | None = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationRequiredData": + return cls( + key=_required_str(value, "key"), + label=_optional_str(value, "label") or _required_str(value, "key"), + data_type=_optional_str(value, "data_type") or _optional_str(value, "type") or "string", + required=_bool(value.get("required"), default=True), + secret=_bool(value.get("secret"), default=False), + description=_optional_str(value, "description"), + ) + + def to_dict(self) -> dict[str, object]: + return { + "key": self.key, + "label": self.label, + "data_type": self.data_type, + "required": self.required, + "secret": self.secret, + "description": self.description, + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationProviderDescription: + module_id: str + fragment_types: tuple[str, ...] = () + schema_refs: Mapping[str, str] = field(default_factory=dict) + exported_scopes: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class ConfigurationPreflightContext: + tenant_id: str | None = None + operator_user_id: str | None = None + supplied_data: Mapping[str, Any] = field(default_factory=dict) + installed_modules: Mapping[str, str] = field(default_factory=dict) + capabilities: frozenset[str] = frozenset() + dry_run: bool = True + + +@dataclass(frozen=True, slots=True) +class ConfigurationPreflightResult: + diagnostics: tuple[ConfigurationDiagnostic, ...] = () + required_data: tuple[ConfigurationRequiredData, ...] = () + plan: tuple[ConfigurationPlanItem, ...] = () + + +@dataclass(frozen=True, slots=True) +class ConfigurationApplyResult: + diagnostics: tuple[ConfigurationDiagnostic, ...] = () + created_refs: Mapping[str, str] = field(default_factory=dict) + updated_refs: Mapping[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class ConfigurationExportSelection: + tenant_id: str | None = None + scopes: tuple[str, ...] = () + module_ids: tuple[str, ...] = () + object_refs: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class ConfigurationExportResult: + fragments: tuple[ConfigurationPackageFragment, ...] = () + data_requirements: tuple[ConfigurationRequiredData, ...] = () + diagnostics: tuple[ConfigurationDiagnostic, ...] = () + + +@runtime_checkable +class ConfigurationProvider(Protocol): + module_id: str + + def describe(self) -> ConfigurationProviderDescription: + ... + + def preflight(self, fragment: ConfigurationPackageFragment, context: ConfigurationPreflightContext) -> ConfigurationPreflightResult: + ... + + def apply(self, fragment: ConfigurationPackageFragment, supplied_data: Mapping[str, Any], context: ConfigurationPreflightContext) -> ConfigurationApplyResult: + ... + + def export(self, selection: ConfigurationExportSelection, context: ConfigurationPreflightContext) -> ConfigurationExportResult: + ... + + def health(self, import_result: ConfigurationApplyResult, context: ConfigurationPreflightContext) -> tuple[ConfigurationDiagnostic, ...]: + ... + + +def dry_run_configuration_package( + package: ConfigurationPackageManifest | Mapping[str, Any], + providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider], + context: ConfigurationPreflightContext, +) -> ConfigurationPreflightResult: + manifest = _configuration_package_manifest(package) + provider_map = _configuration_provider_map(providers) + diagnostics: list[ConfigurationDiagnostic] = [] + required_data: list[ConfigurationRequiredData] = [] + plan: list[ConfigurationPlanItem] = [] + + diagnostics.extend(_module_requirement_diagnostics(manifest, context)) + diagnostics.extend(_capability_requirement_diagnostics(manifest, context)) + for item in manifest.data_requirements: + requirement = ConfigurationRequiredData.from_mapping(item) + required_data.append(requirement) + if requirement.required and requirement.key not in context.supplied_data: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="required_data_missing", + message=f"Required operator data is missing: {requirement.label}.", + object_ref=requirement.key, + resolution="Collect this value in the configuration package wizard before applying.", + )) + + for fragment in manifest.fragments: + provider = provider_map.get(fragment.module_id) + if provider is None: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="configuration_provider_missing", + message=f"No configuration provider is registered for module {fragment.module_id!r}.", + module_id=fragment.module_id, + object_ref=fragment.fragment_id or fragment.fragment_type, + resolution="Install and enable the module version that provides this configuration provider.", + )) + plan.append(ConfigurationPlanItem(action="blocked", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Provider is missing.")) + continue + description = provider.describe() + if description.fragment_types and fragment.fragment_type not in description.fragment_types: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="fragment_type_unsupported", + message=f"Provider {fragment.module_id!r} does not support fragment type {fragment.fragment_type!r}.", + module_id=fragment.module_id, + object_ref=fragment.fragment_id or fragment.fragment_type, + resolution="Use a package version compatible with the installed module provider.", + )) + plan.append(ConfigurationPlanItem(action="blocked", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Fragment type is unsupported.")) + continue + try: + result = provider.preflight(fragment, context) + except Exception as exc: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="provider_preflight_failed", + message=f"Configuration provider {fragment.module_id!r} preflight failed: {exc}", + module_id=fragment.module_id, + object_ref=fragment.fragment_id or fragment.fragment_type, + resolution="Review provider diagnostics and package fragment payload.", + )) + plan.append(ConfigurationPlanItem(action="blocked", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Provider preflight failed.")) + continue + diagnostics.extend(result.diagnostics) + required_data.extend(result.required_data) + for requirement in result.required_data: + if requirement.required and requirement.key not in context.supplied_data: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="required_data_missing", + message=f"Required operator data is missing: {requirement.label}.", + module_id=fragment.module_id, + object_ref=requirement.key, + resolution="Collect this value in the configuration package wizard before applying.", + )) + plan.extend(result.plan or (ConfigurationPlanItem(action="noop", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id, summary="Provider reported no changes."),)) + return ConfigurationPreflightResult( + diagnostics=tuple(_dedupe_diagnostics(diagnostics)), + required_data=tuple(_dedupe_required_data(required_data)), + plan=tuple(plan), + ) + + +def apply_configuration_package( + package: ConfigurationPackageManifest | Mapping[str, Any], + providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider], + context: ConfigurationPreflightContext, + supplied_data: Mapping[str, Any] | None = None, +) -> ConfigurationApplyResult: + manifest = _configuration_package_manifest(package) + apply_context = ConfigurationPreflightContext( + tenant_id=context.tenant_id, + operator_user_id=context.operator_user_id, + supplied_data=supplied_data if supplied_data is not None else context.supplied_data, + installed_modules=context.installed_modules, + capabilities=context.capabilities, + dry_run=False, + ) + preflight = dry_run_configuration_package(manifest, providers, apply_context) + blockers = [item for item in preflight.diagnostics if item.severity == "blocker"] + if blockers: + return ConfigurationApplyResult(diagnostics=tuple(blockers)) + provider_map = _configuration_provider_map(providers) + diagnostics: list[ConfigurationDiagnostic] = list(preflight.diagnostics) + created_refs: dict[str, str] = {} + updated_refs: dict[str, str] = {} + for fragment in manifest.fragments: + provider = provider_map[fragment.module_id] + try: + result = provider.apply(fragment, apply_context.supplied_data, apply_context) + diagnostics.extend(result.diagnostics) + created_refs.update(result.created_refs) + updated_refs.update(result.updated_refs) + diagnostics.extend(provider.health(result, apply_context)) + except Exception as exc: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="provider_apply_failed", + message=f"Configuration provider {fragment.module_id!r} apply failed: {exc}", + module_id=fragment.module_id, + object_ref=fragment.fragment_id or fragment.fragment_type, + resolution="Stop the import, keep previous configuration, and inspect provider logs.", + )) + return ConfigurationApplyResult( + diagnostics=tuple(_dedupe_diagnostics(diagnostics)), + created_refs=created_refs, + updated_refs=updated_refs, + ) + + +def export_configuration_package( + providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider], + selection: ConfigurationExportSelection, + context: ConfigurationPreflightContext, +) -> ConfigurationExportResult: + provider_map = _configuration_provider_map(providers) + module_ids = selection.module_ids or tuple(provider_map) + fragments: list[ConfigurationPackageFragment] = [] + data_requirements: list[ConfigurationRequiredData] = [] + diagnostics: list[ConfigurationDiagnostic] = [] + for module_id in module_ids: + provider = provider_map.get(module_id) + if provider is None: + diagnostics.append(ConfigurationDiagnostic( + severity="warning", + code="configuration_provider_missing", + message=f"No configuration provider is registered for module {module_id!r}; skipping export.", + module_id=module_id, + resolution="Enable the module provider before exporting its configuration.", + )) + continue + try: + result = provider.export(selection, context) + except Exception as exc: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="provider_export_failed", + message=f"Configuration provider {module_id!r} export failed: {exc}", + module_id=module_id, + resolution="Review provider diagnostics before reusing this package export.", + )) + continue + fragments.extend(result.fragments) + data_requirements.extend(result.data_requirements) + diagnostics.extend(result.diagnostics) + return ConfigurationExportResult( + fragments=tuple(fragments), + data_requirements=tuple(_dedupe_required_data(data_requirements)), + diagnostics=tuple(_dedupe_diagnostics(diagnostics)), + ) + + +def configuration_package_catalog( + path: Path | str | None = None, + *, + require_trusted: bool | None = None, + approved_channels: tuple[str, ...] | None = None, + trusted_keys: dict[str, str] | None = None, +) -> tuple[dict[str, object], ...]: + validation = validate_configuration_package_catalog( + path, + require_trusted=require_trusted, + approved_channels=approved_channels, + trusted_keys=trusted_keys, + ) + if not validation["valid"]: + raise ValueError(str(validation["error"] or "Configuration package catalog is invalid.")) + return tuple(item for item in validation["packages"] if isinstance(item, dict)) + + +def validate_configuration_package_catalog( + path: Path | str | None = None, + *, + require_trusted: bool | None = None, + approved_channels: tuple[str, ...] | None = None, + trusted_keys: dict[str, str] | None = None, +) -> dict[str, object]: + source = _catalog_source(path) + configured = source is not None and _catalog_source_exists(source) + if source is not None and not _catalog_source_exists(source): + return _validation_result(source, configured=True, packages=(), error=f"Configuration package catalog does not exist: {source}") + read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() else None} + try: + payload, read_state = _read_catalog_payload_with_metadata(source) + packages = _normalize_catalog_packages(payload) + channel = _catalog_channel(payload) + sequence = _catalog_sequence(payload) + generated_at = _catalog_optional_text(payload, "generated_at") + not_before = _catalog_optional_text(payload, "not_before") + expires_at = _catalog_optional_text(payload, "expires_at") + effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted + effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels + effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys() + signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys) + freshness = _catalog_freshness_state(payload) + replay = _catalog_replay_state(channel=channel, sequence=sequence) + except Exception as exc: + return _validation_result(source, configured=configured, read_state=read_state, packages=(), error=str(exc)) + if signature_state.get("fatal"): + return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"])) + if effective_approved_channels and channel not in effective_approved_channels: + return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=f"Configuration package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.") + if effective_require_trusted and not signature_state["trusted"]: + return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(signature_state["error"] or "Configuration package catalog must be signed by a trusted key.")) + if not freshness["valid"]: + return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(freshness["error"])) + if not replay["valid"]: + return _validation_result(source, configured=configured, read_state=read_state, packages=(), channel=channel, sequence=sequence, generated_at=generated_at, not_before=not_before, expires_at=expires_at, signature_state=signature_state, error=str(replay["error"])) + warnings = [str(item) for item in freshness.get("warnings", ()) if item] + warnings.extend(str(item) for item in replay.get("warnings", ()) if item) + if not signature_state["signed"]: + warnings.append("Configuration package catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.") + elif not signature_state["trusted"]: + warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key.")) + return _validation_result( + source, + valid=True, + configured=configured, + read_state=read_state, + packages=packages, + channel=channel, + sequence=sequence, + generated_at=generated_at, + not_before=not_before, + expires_at=expires_at, + signature_state=signature_state, + warnings=warnings, + ) + + +def sign_configuration_package_catalog(*, path: Path, key_id: str, private_key_path: Path, output_path: Path | None = None) -> Path: + payload = _read_catalog_payload(path) + if not isinstance(payload, dict): + raise ValueError("Only object-style configuration package catalogs can be signed.") + signed_payload = dict(payload) + signed_payload.pop("signature", None) + signed_payload.pop("signatures", None) + private_key = _load_private_key(private_key_path) + signature = private_key.sign(_canonical_catalog_bytes(signed_payload)) + signed_payload["signature"] = { + "algorithm": "ed25519", + "key_id": key_id, + "value": base64.b64encode(signature).decode("ascii"), + } + target = output_path or path + target.write_text(json.dumps(signed_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return target + + +def record_configuration_package_catalog_acceptance(validation: dict[str, object]) -> None: + state_path = _configured_sequence_state_path() + if state_path is None or validation.get("valid") is not True: + return + channel = validation.get("channel") + sequence = validation.get("sequence") + if not isinstance(channel, str) or not isinstance(sequence, int): + return + try: + state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.exists() else {} + except json.JSONDecodeError: + state = {} + if not isinstance(state, dict): + state = {} + channels = state.get("channels") + if not isinstance(channels, dict): + channels = {} + channel_state = channels.get(channel) + if not isinstance(channel_state, dict): + channel_state = {} + channel_state["last_sequence"] = max(int(channel_state.get("last_sequence") or 0), sequence) + channel_state["accepted_at"] = datetime.now(tz=UTC).isoformat().replace("+00:00", "Z") + channel_state["key_id"] = validation.get("key_id") + channel_state["source"] = validation.get("source") or validation.get("path") + channels[channel] = channel_state + state["channels"] = channels + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _configuration_package_manifest(package: ConfigurationPackageManifest | Mapping[str, Any]) -> ConfigurationPackageManifest: + if isinstance(package, ConfigurationPackageManifest): + return package + return ConfigurationPackageManifest.from_mapping(package) + + +def _configuration_provider_map(providers: Sequence[ConfigurationProvider] | Mapping[str, ConfigurationProvider]) -> dict[str, ConfigurationProvider]: + if isinstance(providers, Mapping): + return {str(key): provider for key, provider in providers.items()} + return {provider.module_id: provider for provider in providers} + + +def _module_requirement_diagnostics(manifest: ConfigurationPackageManifest, context: ConfigurationPreflightContext) -> list[ConfigurationDiagnostic]: + diagnostics: list[ConfigurationDiagnostic] = [] + installed = context.installed_modules + for requirement in manifest.required_modules: + installed_version = installed.get(requirement.module_id) + if installed_version is None: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="required_module_missing", + message=f"Required module {requirement.module_id!r} is not installed.", + module_id=requirement.module_id, + resolution="Install and enable the required module before applying this configuration package.", + )) + elif requirement.version and installed_version != requirement.version: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="required_module_version_mismatch", + message=f"Required module {requirement.module_id!r} needs version {requirement.version}, but {installed_version} is installed.", + module_id=requirement.module_id, + resolution="Install a compatible module version or use a matching configuration package.", + )) + return diagnostics + + +def _capability_requirement_diagnostics(manifest: ConfigurationPackageManifest, context: ConfigurationPreflightContext) -> list[ConfigurationDiagnostic]: + diagnostics: list[ConfigurationDiagnostic] = [] + capabilities = set(context.capabilities) + for capability in manifest.required_capabilities: + if capability not in capabilities: + diagnostics.append(ConfigurationDiagnostic( + severity="blocker", + code="required_capability_missing", + message=f"Required capability {capability!r} is not available.", + object_ref=capability, + resolution="Enable the module or platform capability before applying this configuration package.", + )) + return diagnostics + + +def _dedupe_diagnostics(items: Sequence[ConfigurationDiagnostic]) -> list[ConfigurationDiagnostic]: + seen: set[tuple[object, ...]] = set() + result: list[ConfigurationDiagnostic] = [] + for item in items: + key = (item.severity, item.code, item.module_id, item.object_ref, item.message) + if key in seen: + continue + seen.add(key) + result.append(item) + return result + + +def _dedupe_required_data(items: Sequence[ConfigurationRequiredData]) -> list[ConfigurationRequiredData]: + seen: set[str] = set() + result: list[ConfigurationRequiredData] = [] + for item in items: + if item.key in seen: + continue + seen.add(item.key) + result.append(item) + return result + + +def _catalog_source(path: Path | str | None) -> Path | str | None: + if path is not None: + return path if isinstance(path, str) and _is_http_url(path) else Path(path).expanduser() + url = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_URL", "").strip() + if url: + return url + value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG", "").strip() + return Path(value).expanduser() if value else None + + +def _configured_catalog_cache_path() -> Path | None: + value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_CACHE", "").strip() + return Path(value).expanduser() if value else None + + +def _configured_sequence_state_path() -> Path | None: + value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_SEQUENCE_STATE", "").strip() + return Path(value).expanduser() if value else None + + +def _configured_enforce_sequence() -> bool: + return os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_ENFORCE_SEQUENCE", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _configured_require_signature() -> bool: + return os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_REQUIRE_SIGNATURE", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _configured_approved_channels() -> tuple[str, ...]: + value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_APPROVED_CHANNELS", "").strip() + return tuple(item.strip() for item in value.split(",") if item.strip()) if value else () + + +def _configured_trusted_keys() -> dict[str, str]: + file_value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "").strip() + if file_value: + path = Path(file_value).expanduser() + if not path.exists(): + raise ValueError(f"Trusted configuration catalog key file does not exist: {path}") + return _parse_trusted_keys(path.read_text(encoding="utf-8")) + url_value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_URL", "").strip() + if url_value: + return _parse_trusted_keys(_read_trusted_keys_url(url_value)) + value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS", "").strip() + return _parse_trusted_keys(value) if value else {} + + +def _configured_trusted_keys_cache_path() -> Path | None: + value = os.getenv("GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE", "").strip() + return Path(value).expanduser() if value else None + + +def _read_trusted_keys_url(url: str) -> str: + if not _is_http_url(url): + raise ValueError("Trusted configuration catalog key URL must use http:// or https://.") + cache_path = _configured_trusted_keys_cache_path() + try: + with urllib.request.urlopen(url, timeout=15) as response: + body = response.read().decode("utf-8") + if cache_path is not None: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body, encoding="utf-8") + return body + except (OSError, urllib.error.URLError): + if cache_path is not None and cache_path.exists(): + return cache_path.read_text(encoding="utf-8") + raise + + +def _read_catalog_payload(source: Path | str | None) -> object: + payload, _metadata = _read_catalog_payload_with_metadata(source) + return payload + + +def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[object, dict[str, object]]: + cache_path = _configured_catalog_cache_path() + metadata: dict[str, object] = {"cache_used": False, "cache_path": str(cache_path) if cache_path else None} + if source is None: + return {"packages": []}, metadata + if isinstance(source, str) and _is_http_url(source): + try: + with urllib.request.urlopen(source, timeout=15) as response: + body = response.read().decode("utf-8") + if cache_path is not None: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body, encoding="utf-8") + return json.loads(body), metadata + except (OSError, urllib.error.URLError): + if cache_path is not None and cache_path.exists(): + metadata["cache_used"] = True + return json.loads(cache_path.read_text(encoding="utf-8")), metadata + raise + return json.loads(Path(source).read_text(encoding="utf-8")), metadata + + +def _normalize_catalog_packages(payload: object) -> tuple[dict[str, object], ...]: + raw_items = payload.get("packages") if isinstance(payload, dict) else payload + if not isinstance(raw_items, list): + raise ValueError("Configuration package catalog must be a list or an object with a 'packages' list.") + return tuple(ConfigurationPackageManifest.from_mapping(item).to_dict() for item in raw_items if isinstance(item, Mapping)) + + +def _catalog_channel(payload: object) -> str | None: + if not isinstance(payload, dict): + return None + value = payload.get("channel") + return str(value).strip() if value is not None and str(value).strip() else None + + +def _catalog_replay_state(*, channel: str | None, sequence: int | None) -> dict[str, object]: + state_path = _configured_sequence_state_path() + if state_path is None: + return {"valid": True, "warnings": ()} + if not channel: + return {"valid": False, "error": "Configuration package catalog sequence replay protection needs a channel."} + if sequence is None: + return {"valid": False, "error": "Configuration package catalog sequence replay protection needs a sequence."} + if not state_path.exists(): + return {"valid": True, "warnings": ()} + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {"valid": False, "error": f"Configuration catalog sequence state file is not valid JSON: {state_path}"} + channels = state.get("channels") if isinstance(state, dict) else None + channel_state = channels.get(channel) if isinstance(channels, dict) else None + last_sequence = channel_state.get("last_sequence") if isinstance(channel_state, dict) else None + if last_sequence is None: + return {"valid": True, "warnings": ()} + last = int(last_sequence) + if sequence < last: + return {"valid": False, "error": f"Configuration package catalog sequence {sequence} is older than accepted sequence {last} for channel {channel!r}."} + if sequence == last and _configured_enforce_sequence(): + return {"valid": False, "error": f"Configuration package catalog sequence {sequence} was already accepted for channel {channel!r}."} + return {"valid": True, "warnings": ()} + + +def _validation_result( + source: Path | str | None, + *, + valid: bool = False, + configured: bool = False, + read_state: Mapping[str, object] | None = None, + packages: Sequence[Mapping[str, object]] = (), + channel: str | None = None, + sequence: int | None = None, + generated_at: str | None = None, + not_before: str | None = None, + expires_at: str | None = None, + signature_state: Mapping[str, object] | None = None, + warnings: Sequence[str] = (), + error: str | None = None, +) -> dict[str, object]: + state = signature_state or {"signed": False, "trusted": False, "key_id": None} + read_state = read_state or {} + return { + "valid": valid, + "configured": configured, + "path": str(source) if source is not None else None, + "source": str(source) if source is not None else None, + "source_type": _catalog_source_type(source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, + "packages": [dict(item) for item in packages], + "channel": channel, + "sequence": sequence, + "generated_at": generated_at, + "not_before": not_before, + "expires_at": expires_at, + "signed": bool(state.get("signed")), + "trusted": bool(state.get("trusted")), + "key_id": state.get("key_id"), + "warnings": list(warnings), + "error": error, + } + + +def _module_requirements(value: object, *, optional: bool) -> list[ConfigurationModuleRequirement]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("Configuration package module requirements must be lists.") + result: list[ConfigurationModuleRequirement] = [] + for item in value: + if isinstance(item, str): + result.append(ConfigurationModuleRequirement(module_id=item, optional=optional)) + elif isinstance(item, Mapping): + result.append(ConfigurationModuleRequirement(module_id=_required_str(item, "module_id"), version=_optional_str(item, "version"), optional=optional)) + else: + raise ValueError("Configuration package module requirements must be strings or objects.") + return result + + +def _fragments(value: object) -> list[ConfigurationPackageFragment]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("Configuration package fragments must be a list.") + result: list[ConfigurationPackageFragment] = [] + for item in value: + if not isinstance(item, Mapping): + raise ValueError("Configuration package fragments must be objects.") + payload = item.get("payload") + result.append(ConfigurationPackageFragment( + module_id=_required_str(item, "module_id"), + fragment_type=_required_str(item, "fragment_type"), + fragment_id=_optional_str(item, "fragment_id"), + payload=payload if isinstance(payload, Mapping) else {}, + )) + return result + + +def _object_list(value: object, *, field_name: str) -> list[dict[str, object]]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"Configuration package {field_name} must be a list.") + return [dict(item) for item in value if isinstance(item, Mapping)] + + +def _string_list(value: object) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("Configuration package list fields must be lists.") + return [str(item).strip() for item in value if str(item).strip()] + + +def _bool(value: object, *, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + return str(value).strip().casefold() in {"1", "true", "yes", "on"} + + +def _required_str(value: Mapping[str, Any], key: str) -> str: + item = _optional_str(value, key) + if not item: + raise ValueError(f"Configuration package entry is missing {key!r}.") + return item + + +def _optional_str(value: Mapping[str, Any], key: str) -> str | None: + item = value.get(key) + if item is None: + return None + text = str(item).strip() + return text or None diff --git a/src/govoplan_core/core/configuration_safety.py b/src/govoplan_core/core/configuration_safety.py new file mode 100644 index 0000000..ce03ddf --- /dev/null +++ b/src/govoplan_core/core/configuration_safety.py @@ -0,0 +1,395 @@ +from __future__ import annotations + +from dataclasses import dataclass +from collections.abc import Mapping +from typing import Any, Literal + +from govoplan_core.security.permissions import scopes_grant + + +ConfigurationScope = Literal["system", "tenant", "user", "group", "campaign"] +ConfigurationRisk = Literal["low", "medium", "high", "destructive"] +SecretHandling = Literal["none", "reference_only", "env_only"] + + +@dataclass(frozen=True, slots=True) +class ConfigurationFieldSafety: + key: str + label: str + owner_module: str + scope: ConfigurationScope + storage: str + ui_managed: bool + risk: ConfigurationRisk + secret_handling: SecretHandling = "none" + required_scopes: tuple[str, ...] = () + dry_run_required: bool = False + validation_required: bool = True + policy_explanation_required: bool = False + audit_event: str | None = None + maintenance_required: bool = False + two_person_approval_required: bool = False + rollback_history_required: bool = False + notes: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "key": self.key, + "label": self.label, + "owner_module": self.owner_module, + "scope": self.scope, + "storage": self.storage, + "ui_managed": self.ui_managed, + "risk": self.risk, + "secret_handling": self.secret_handling, + "required_scopes": list(self.required_scopes), + "dry_run_required": self.dry_run_required, + "validation_required": self.validation_required, + "policy_explanation_required": self.policy_explanation_required, + "audit_event": self.audit_event, + "maintenance_required": self.maintenance_required, + "two_person_approval_required": self.two_person_approval_required, + "rollback_history_required": self.rollback_history_required, + "notes": self.notes, + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationChangeSafetyPlan: + key: str + allowed: bool + field: ConfigurationFieldSafety | None + risk: ConfigurationRisk | None = None + missing_scopes: tuple[str, ...] = () + dry_run_required: bool = False + dry_run_satisfied: bool = False + approval_required: bool = False + approval_satisfied: bool = False + maintenance_required: bool = False + maintenance_satisfied: bool = False + rollback_history_required: bool = False + secret_handling: SecretHandling = "none" + audit_event: str | None = None + policy_explanation: str | None = None + blockers: tuple[str, ...] = () + warnings: tuple[str, ...] = () + + def to_dict(self) -> dict[str, object]: + return { + "key": self.key, + "allowed": self.allowed, + "field": self.field.to_dict() if self.field else None, + "risk": self.risk, + "missing_scopes": list(self.missing_scopes), + "dry_run_required": self.dry_run_required, + "dry_run_satisfied": self.dry_run_satisfied, + "approval_required": self.approval_required, + "approval_satisfied": self.approval_satisfied, + "maintenance_required": self.maintenance_required, + "maintenance_satisfied": self.maintenance_satisfied, + "rollback_history_required": self.rollback_history_required, + "secret_handling": self.secret_handling, + "audit_event": self.audit_event, + "policy_explanation": self.policy_explanation, + "blockers": list(self.blockers), + "warnings": list(self.warnings), + } + + +_CONFIGURATION_FIELD_SAFETY: tuple[ConfigurationFieldSafety, ...] = ( + ConfigurationFieldSafety( + key="module_management.desired_enabled", + label="Enabled modules", + owner_module="core", + scope="system", + storage="system_settings", + ui_managed=True, + risk="high", + required_scopes=("system:settings:write",), + dry_run_required=True, + policy_explanation_required=True, + audit_event="module_management.updated", + maintenance_required=True, + rollback_history_required=True, + notes="Changing enabled modules must use module preflight and installer rollback records.", + ), + ConfigurationFieldSafety( + key="module_management.install_plan", + label="Module package install plan", + owner_module="core", + scope="system", + storage="system_settings", + ui_managed=True, + risk="destructive", + required_scopes=("system:settings:write",), + dry_run_required=True, + policy_explanation_required=True, + audit_event="module_install.requested", + maintenance_required=True, + two_person_approval_required=True, + rollback_history_required=True, + notes="Package mutation needs maintenance mode, dry-run preflight, approval, and run records.", + ), + ConfigurationFieldSafety( + key="configuration_packages.apply", + label="Configuration package apply", + owner_module="core", + scope="system", + storage="configuration_packages", + ui_managed=True, + risk="high", + required_scopes=("system:settings:write",), + dry_run_required=True, + policy_explanation_required=True, + audit_event="configuration_package.applied", + maintenance_required=True, + two_person_approval_required=True, + rollback_history_required=True, + notes="Configuration package apply needs a dry-run, maintenance mode, approval, and version history.", + ), + ConfigurationFieldSafety( + key="maintenance_mode", + label="Maintenance mode", + owner_module="core", + scope="system", + storage="system_settings", + ui_managed=True, + risk="high", + required_scopes=("system:settings:write",), + validation_required=True, + audit_event="system_settings.updated", + rollback_history_required=True, + notes="Maintenance mode controls platform availability and gates dangerous operations.", + ), + ConfigurationFieldSafety( + key="privacy_retention_policy", + label="Privacy retention policy", + owner_module="core", + scope="system", + storage="system_settings", + ui_managed=True, + risk="high", + required_scopes=("system:settings:write", "admin:policies:write"), + dry_run_required=True, + policy_explanation_required=True, + audit_event="privacy_retention.policy_updated", + two_person_approval_required=True, + rollback_history_required=True, + notes="Retention reductions and destructive cleanup require explainable policy decisions and dry-run counts.", + ), + ConfigurationFieldSafety( + key="governance_templates", + label="Governance templates", + owner_module="access", + scope="system", + storage="governance_templates", + ui_managed=True, + risk="high", + required_scopes=("system:governance:write",), + dry_run_required=True, + policy_explanation_required=True, + audit_event="governance_template.updated", + two_person_approval_required=True, + rollback_history_required=True, + notes="Central groups, roles, and assignments need previews before materialization.", + ), + ConfigurationFieldSafety( + key="mail_profiles.credentials", + label="Mail profile credentials", + owner_module="mail", + scope="tenant", + storage="module_settings", + ui_managed=True, + risk="high", + secret_handling="reference_only", + required_scopes=("mail_servers:manage_credentials",), + validation_required=True, + audit_event="mail_server_profile.credential_updated", + two_person_approval_required=True, + rollback_history_required=True, + notes="UI must store secret references/placeholders only; secret values are never echoed.", + ), + ConfigurationFieldSafety( + key="files.connector_profiles", + label="File connector profiles", + owner_module="files", + scope="tenant", + storage="module_settings", + ui_managed=True, + risk="high", + secret_handling="reference_only", + required_scopes=("files:file:admin",), + dry_run_required=True, + policy_explanation_required=True, + audit_event="files.connector_profile.updated", + two_person_approval_required=True, + rollback_history_required=True, + notes="Connector endpoints are UI-manageable, but passwords/tokens remain env or secret refs.", + ), + ConfigurationFieldSafety( + key="DATABASE_URL", + label="Database URL", + owner_module="core", + scope="system", + storage="environment", + ui_managed=False, + risk="destructive", + secret_handling="env_only", + maintenance_required=True, + notes="Database connectivity remains deployment-managed and must not be changed from the running UI.", + ), + ConfigurationFieldSafety( + key="MASTER_KEY_B64", + label="Master encryption key", + owner_module="core", + scope="system", + storage="environment", + ui_managed=False, + risk="destructive", + secret_handling="env_only", + maintenance_required=True, + two_person_approval_required=True, + notes="Encryption roots remain out of band; UI may only report missing/rotated state.", + ), + ConfigurationFieldSafety( + key="GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS", + label="Module package catalog trusted keys", + owner_module="core", + scope="system", + storage="environment", + ui_managed=False, + risk="high", + secret_handling="env_only", + notes="Trust roots are deployment-managed; UI can validate catalogs but should not edit key material.", + ), + ConfigurationFieldSafety( + key="GOVOPLAN_CONFIGURATION_PACKAGE_CATALOG_TRUSTED_KEYS", + label="Configuration package catalog trusted keys", + owner_module="core", + scope="system", + storage="environment", + ui_managed=False, + risk="high", + secret_handling="env_only", + notes="Configuration package trust roots are deployment-managed.", + ), +) + + +def configuration_safety_catalog(*, include_env_only: bool = True) -> tuple[ConfigurationFieldSafety, ...]: + if include_env_only: + return _CONFIGURATION_FIELD_SAFETY + return tuple(item for item in _CONFIGURATION_FIELD_SAFETY if item.ui_managed) + + +def classify_configuration_field(key: str) -> ConfigurationFieldSafety | None: + clean = key.strip() + for item in _CONFIGURATION_FIELD_SAFETY: + if item.key == clean: + return item + return None + + +def high_impact_configuration_fields() -> tuple[ConfigurationFieldSafety, ...]: + return tuple(item for item in _CONFIGURATION_FIELD_SAFETY if item.risk in {"high", "destructive"}) + + +def ui_managed_configuration_fields_requiring_approval() -> tuple[ConfigurationFieldSafety, ...]: + return tuple(item for item in _CONFIGURATION_FIELD_SAFETY if item.ui_managed and item.two_person_approval_required) + + +def plan_configuration_change( + key: str, + *, + actor_scopes: tuple[str, ...] | list[str] = (), + value: object = None, + dry_run: bool = False, + maintenance_mode: bool = False, + approval_count: int = 0, + include_env_only: bool = True, +) -> ConfigurationChangeSafetyPlan: + field = classify_configuration_field(key) + if field is None: + return ConfigurationChangeSafetyPlan( + key=key, + allowed=False, + field=None, + blockers=("unknown_configuration_field",), + policy_explanation="This setting is not in the configuration safety catalog.", + ) + if not include_env_only and not field.ui_managed: + return ConfigurationChangeSafetyPlan( + key=key, + allowed=False, + field=field, + risk=field.risk, + secret_handling=field.secret_handling, + blockers=("deployment_managed",), + policy_explanation="This setting is deployment-managed and cannot be edited through the UI.", + ) + blockers: list[str] = [] + warnings: list[str] = [] + missing_scopes = tuple(scope for scope in field.required_scopes if not scopes_grant(actor_scopes, scope)) + if missing_scopes: + blockers.append("missing_required_scope") + if not field.ui_managed: + blockers.append("deployment_managed") + if field.dry_run_required and not dry_run: + blockers.append("dry_run_required") + if field.maintenance_required and not maintenance_mode: + blockers.append("maintenance_mode_required") + approval_required = field.two_person_approval_required + approval_satisfied = not approval_required or approval_count >= 2 + if approval_required and not approval_satisfied: + blockers.append("two_person_approval_required") + if field.secret_handling == "reference_only" and _contains_plain_secret(value): + blockers.append("secret_reference_required") + if field.secret_handling == "env_only" and value is not None: + blockers.append("env_only_secret") + if field.rollback_history_required: + warnings.append("rollback_history_required") + return ConfigurationChangeSafetyPlan( + key=field.key, + allowed=not blockers, + field=field, + risk=field.risk, + missing_scopes=missing_scopes, + dry_run_required=field.dry_run_required, + dry_run_satisfied=not field.dry_run_required or dry_run, + approval_required=approval_required, + approval_satisfied=approval_satisfied, + maintenance_required=field.maintenance_required, + maintenance_satisfied=not field.maintenance_required or maintenance_mode, + rollback_history_required=field.rollback_history_required, + secret_handling=field.secret_handling, + audit_event=field.audit_event, + policy_explanation=_policy_explanation(field), + blockers=tuple(dict.fromkeys(blockers)), + warnings=tuple(dict.fromkeys(warnings)), + ) + + +def _policy_explanation(field: ConfigurationFieldSafety) -> str: + parts = [f"{field.label} is {field.risk}-risk"] + if field.dry_run_required: + parts.append("requires dry-run preview") + if field.two_person_approval_required: + parts.append("requires two-person approval") + if field.maintenance_required: + parts.append("requires maintenance mode") + if field.secret_handling != "none": + parts.append(f"uses {field.secret_handling} secret handling") + return "; ".join(parts) + "." + + +def _contains_plain_secret(value: object) -> bool: + if not isinstance(value, Mapping): + return False + secret_keys = {"password", "token", "secret", "api_key", "access_key", "secret_key"} + for key, item in value.items(): + name = str(key).strip().casefold() + if name in secret_keys and item not in (None, "", {"secret_ref": ""}): + return True + if isinstance(item, Mapping) and _contains_plain_secret(item): + return True + return False diff --git a/src/govoplan_core/core/events.py b/src/govoplan_core/core/events.py index cfcad49..d5fe16d 100644 --- a/src/govoplan_core/core/events.py +++ b/src/govoplan_core/core/events.py @@ -2,9 +2,36 @@ from __future__ import annotations from collections import defaultdict from collections.abc import Callable, Mapping +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass, field from datetime import datetime, timezone +import re from typing import Any +import uuid + + +_TRACE_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") + + +def new_event_id() -> str: + return str(uuid.uuid4()) + + +def normalize_trace_id(value: str | None) -> str | None: + if value is None: + return None + candidate = value.strip() + return candidate if _TRACE_ID_RE.fullmatch(candidate) else None + + +@dataclass(frozen=True, slots=True) +class EventTrace: + correlation_id: str + causation_id: str | None = None + + +_current_trace: ContextVar[EventTrace | None] = ContextVar("govoplan_event_trace", default=None) @dataclass(frozen=True, slots=True) @@ -13,11 +40,59 @@ class PlatformEvent: module_id: str payload: Mapping[str, Any] = field(default_factory=dict) occurred_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + event_id: str = field(default_factory=new_event_id) + correlation_id: str | None = None + causation_id: str | None = None EventHandler = Callable[[PlatformEvent], None] +def current_event_trace() -> EventTrace | None: + return _current_trace.get() + + +@contextmanager +def event_context( + *, + correlation_id: str | None = None, + causation_id: str | None = None, +): + parent = current_event_trace() + trace = EventTrace( + correlation_id=normalize_trace_id(correlation_id) or (parent.correlation_id if parent else new_event_id()), + causation_id=normalize_trace_id(causation_id) or (parent.causation_id if parent else None), + ) + token = _current_trace.set(trace) + try: + yield trace + finally: + _current_trace.reset(token) + + +def event_trace_for(event: PlatformEvent) -> EventTrace: + active = current_event_trace() + return EventTrace( + correlation_id=normalize_trace_id(event.correlation_id) or (active.correlation_id if active else event.event_id), + causation_id=normalize_trace_id(event.causation_id) or (active.causation_id if active else None), + ) + + +def ensure_event_trace(event: PlatformEvent) -> PlatformEvent: + trace = event_trace_for(event) + if event.correlation_id == trace.correlation_id and event.causation_id == trace.causation_id: + return event + return PlatformEvent( + type=event.type, + module_id=event.module_id, + payload=event.payload, + occurred_at=event.occurred_at, + event_id=event.event_id, + correlation_id=trace.correlation_id, + causation_id=trace.causation_id, + ) + + class EventBus: def __init__(self) -> None: self._subscribers: dict[str, list[EventHandler]] = defaultdict(list) @@ -26,8 +101,9 @@ class EventBus: self._subscribers[event_type].append(handler) def publish(self, event: PlatformEvent) -> None: - for handler in self._subscribers.get(event.type, ()): - handler(event) - for handler in self._subscribers.get("*", ()): - handler(event) - + traced = ensure_event_trace(event) + with event_context(correlation_id=traced.correlation_id, causation_id=traced.event_id): + for handler in self._subscribers.get(traced.type, ()): + handler(traced) + for handler in self._subscribers.get("*", ()): + handler(traced) diff --git a/src/govoplan_core/core/identity.py b/src/govoplan_core/core/identity.py new file mode 100644 index 0000000..f6e31ad --- /dev/null +++ b/src/govoplan_core/core/identity.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Literal, Protocol, runtime_checkable + + +IDENTITY_MODULE_ID = "identity" +CAPABILITY_IDENTITY_DIRECTORY = "identity.directory" + +IdentityStatus = Literal["active", "inactive", "suspended"] + + +@dataclass(frozen=True, slots=True) +class IdentityRef: + id: str + display_name: str | None = None + external_subject: str | None = None + source: str = "local" + primary_account_id: str | None = None + account_ids: tuple[str, ...] = () + status: IdentityStatus = "active" + + +@dataclass(frozen=True, slots=True) +class IdentityAccountLinkRef: + id: str + identity_id: str + account_id: str + is_primary: bool = False + source: str = "local" + + +@runtime_checkable +class IdentityDirectory(Protocol): + def get_identity(self, identity_id: str) -> IdentityRef | None: + ... + + def identity_for_account(self, account_id: str) -> IdentityRef | None: + ... + + def identities_for_accounts(self, account_ids: Sequence[str]) -> Sequence[IdentityRef]: + ... + + def accounts_for_identity(self, identity_id: str) -> Sequence[IdentityAccountLinkRef]: + ... diff --git a/src/govoplan_core/core/lifecycle.py b/src/govoplan_core/core/lifecycle.py index f1491d3..08850ed 100644 --- a/src/govoplan_core/core/lifecycle.py +++ b/src/govoplan_core/core/lifecycle.py @@ -10,6 +10,7 @@ from govoplan_core.core.module_management import ModuleManagementError, REQUIRED from govoplan_core.core.modules import ModuleContext, ModuleManifest from govoplan_core.core.registry import PlatformRegistry from govoplan_core.core.runtime import configure_runtime +from govoplan_core.server.route_validation import validate_router_can_mount @dataclass(frozen=True, slots=True) @@ -129,6 +130,7 @@ class ModuleLifecycleManager: module_router = manifest.route_factory(self.context) guarded = APIRouter(dependencies=[Depends(require_module_active(module_id))]) guarded.include_router(module_router) + validate_router_can_mount(app, guarded, prefix=self.api_prefix, owner=f"module {module_id!r}") app.include_router(guarded, prefix=self.api_prefix) app.openapi_schema = None self._mounted_modules.add(module_id) diff --git a/src/govoplan_core/core/module_installer.py b/src/govoplan_core/core/module_installer.py index b612a04..c37d4bd 100644 --- a/src/govoplan_core/core/module_installer.py +++ b/src/govoplan_core/core/module_installer.py @@ -1,10 +1,11 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from contextlib import AbstractContextManager +from contextlib import AbstractContextManager, closing from dataclasses import dataclass from datetime import UTC, datetime from importlib import metadata +import hashlib import json import os from pathlib import Path @@ -22,6 +23,7 @@ import urllib.request from sqlalchemy.orm import Session from govoplan_core.core.maintenance import saved_maintenance_mode +from govoplan_core.core.events import current_event_trace from govoplan_core.core.module_management import ( PROTECTED_MODULES, ModuleInstallPlan, @@ -85,6 +87,7 @@ class ModuleInstallerPreflight: rollback_commands: tuple[str, ...] issues: tuple[ModuleInstallerIssue, ...] checklist: tuple[ModuleInstallChecklistItem, ...] = () + artifact_integrity: tuple[dict[str, object], ...] = () def as_dict(self) -> dict[str, object]: return { @@ -96,6 +99,7 @@ class ModuleInstallerPreflight: "rollback_commands": list(self.rollback_commands), "issues": [issue.as_dict() for issue in self.issues], "checklist": [item.as_dict() for item in self.checklist], + "artifact_integrity": list(self.artifact_integrity), } @@ -203,6 +207,12 @@ def module_install_preflight( if item.webui_package or item.webui_ref: frontend_rebuild_required = True + artifact_integrity, artifact_integrity_issues = _verify_artifact_integrity( + planned_items, + require_verified=_configured_require_artifact_integrity(), + ) + issues.extend(artifact_integrity_issues) + if frontend_rebuild_required: if webui_root is None: issues.append(ModuleInstallerIssue("blocker", "webui_root_missing", "WebUI package changes need a webui root for npm install/build.")) @@ -229,6 +239,7 @@ def module_install_preflight( rollback_commands=rollback_commands, issues=tuple(issues), checklist=checklist, + artifact_integrity=artifact_integrity, ) @@ -251,6 +262,7 @@ def run_module_install_plan( activate_installed_modules: bool = True, remove_uninstalled_modules_from_desired: bool = True, dry_run: bool = False, + request_context: Mapping[str, object] | None = None, ) -> ModuleInstallerRunResult: maintenance_mode = saved_maintenance_mode(session) effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url) @@ -302,6 +314,8 @@ def run_module_install_plan( "activate_installed_modules": activate_installed_modules, "remove_uninstalled_modules_from_desired": remove_uninstalled_modules_from_desired, } + if request_context: + record["request"] = dict(request_context) _write_json(record_path, record) if dry_run: @@ -409,6 +423,7 @@ def supervise_module_install_plan( health_urls: Iterable[str] | None = None, health_timeout_seconds: float = 60.0, health_interval_seconds: float = 2.0, + request_context: Mapping[str, object] | None = None, ) -> ModuleInstallerRunResult: effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url) effective_restart_commands = _command_list(restart_command, restart_commands) @@ -431,6 +446,7 @@ def supervise_module_install_plan( activate_installed_modules=activate_installed_modules, remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, dry_run=False, + request_context=request_context, ) supervisor: dict[str, object] = { "started_at": datetime.now(tz=UTC).isoformat(), @@ -647,8 +663,10 @@ def queue_module_installer_request( options: Mapping[str, object] | None = None, requested_by: str | None = None, retry_of: str | None = None, + trace: Mapping[str, object] | None = None, ) -> dict[str, object]: request_id = _run_id() + effective_trace = dict(trace) if trace is not None else _current_trace_payload() record = { "request_id": request_id, "status": "queued", @@ -656,6 +674,8 @@ def queue_module_installer_request( "requested_by": requested_by, "options": dict(options or {}), } + if effective_trace: + record["trace"] = effective_trace if retry_of: record["retry_of"] = retry_of with _request_queue_lock(runtime_dir): @@ -1273,6 +1293,7 @@ def _run_summary(path: Path, record: Mapping[str, object]) -> dict[str, object]: supervisor = record.get("supervisor") if isinstance(record.get("supervisor"), Mapping) else {} commands = record.get("commands") if isinstance(record.get("commands"), list) else [] plan = record.get("plan") if isinstance(record.get("plan"), list) else [] + request = record.get("request") if isinstance(record.get("request"), Mapping) else {} return { "run_id": str(record.get("run_id") or path.parent.name), "status": str(record.get("status") or "unknown"), @@ -1284,9 +1305,22 @@ def _run_summary(path: Path, record: Mapping[str, object]) -> dict[str, object]: "record_path": str(path), "commands_count": len(commands), "planned_modules": [str(item.get("module_id")) for item in plan if isinstance(item, Mapping) and item.get("module_id")], + "request_id": request.get("request_id") if isinstance(request, Mapping) else None, + "requested_by": request.get("requested_by") if isinstance(request, Mapping) else None, + "trace": request.get("trace") if isinstance(request.get("trace"), Mapping) else None, } +def _current_trace_payload() -> dict[str, object]: + trace = current_event_trace() + if trace is None: + return {} + payload: dict[str, object] = {"correlation_id": trace.correlation_id} + if trace.causation_id: + payload["causation_id"] = trace.causation_id + return payload + + def _database_restore_was_applied(record: Mapping[str, object]) -> bool: restore = record.get("rollback_database_restore") return isinstance(restore, Mapping) and restore.get("restored") is True @@ -1305,6 +1339,156 @@ def _looks_pinned_dependency_ref(value: str) -> bool: return False +def _configured_require_artifact_integrity() -> bool: + return os.getenv("GOVOPLAN_MODULE_INSTALLER_REQUIRE_ARTIFACT_INTEGRITY", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _verify_artifact_integrity( + planned_items: tuple[ModuleInstallPlanItem, ...], + *, + require_verified: bool, +) -> tuple[tuple[dict[str, object], ...], tuple[ModuleInstallerIssue, ...]]: + records: list[dict[str, object]] = [] + issues: list[ModuleInstallerIssue] = [] + for item in planned_items: + if item.action != "install": + continue + for kind, package_name, package_ref in ( + ("python", item.python_package, item.python_ref), + ("webui", item.webui_package, item.webui_ref), + ): + if not package_ref: + continue + raw_metadata = _artifact_metadata(item.artifact_integrity, kind) + if raw_metadata is None: + if require_verified: + issues.append(ModuleInstallerIssue( + "blocker", + "artifact_integrity_required", + f"{kind.capitalize()} artifact integrity metadata is required in production mode.", + item.module_id, + )) + continue + record, record_issues = _verify_artifact_metadata( + item, + kind=kind, + package_name=package_name, + package_ref=package_ref, + metadata=raw_metadata, + require_verified=require_verified, + ) + records.append(record) + issues.extend(record_issues) + return tuple(records), tuple(issues) + + +def _artifact_metadata(value: Mapping[str, object] | None, kind: str) -> Mapping[str, object] | None: + if not isinstance(value, Mapping): + return None + raw = value.get(kind) + return raw if isinstance(raw, Mapping) else None + + +def _verify_artifact_metadata( + item: ModuleInstallPlanItem, + *, + kind: str, + package_name: str | None, + package_ref: str, + metadata: Mapping[str, object], + require_verified: bool, +) -> tuple[dict[str, object], tuple[ModuleInstallerIssue, ...]]: + issues: list[ModuleInstallerIssue] = [] + expected_sha256 = _artifact_text(metadata, "sha256") + artifact_path = _artifact_path(metadata) + expected_ref = _artifact_text(metadata, "ref") or _artifact_text(metadata, "expected_ref") + record: dict[str, object] = { + "module_id": item.module_id, + "kind": kind, + "package_ref": package_ref, + "verified": False, + } + if package_name: + record["package"] = package_name + for key in ("sha256", "sbom_url", "provenance_url", "registry_identity", "git_ref"): + value = _artifact_text(metadata, key) + if value: + record[key] = value + if expected_ref: + record["expected_ref"] = expected_ref + if expected_ref != package_ref: + issues.append(ModuleInstallerIssue( + "blocker", + "artifact_ref_mismatch", + f"{kind.capitalize()} artifact metadata expected ref {expected_ref!r} but the install plan uses {package_ref!r}.", + item.module_id, + )) + if not expected_sha256: + issues.append(ModuleInstallerIssue( + "blocker" if require_verified else "warning", + "artifact_digest_missing", + f"{kind.capitalize()} artifact integrity metadata has no sha256 digest.", + item.module_id, + )) + return record, tuple(issues) + if artifact_path is None: + issues.append(ModuleInstallerIssue( + "blocker" if require_verified else "warning", + "artifact_not_verified", + f"{kind.capitalize()} artifact digest is declared but no local artifact path is available for verification.", + item.module_id, + )) + return record, tuple(issues) + record["artifact_path"] = str(artifact_path) + if not artifact_path.exists(): + issues.append(ModuleInstallerIssue( + "blocker", + "artifact_missing", + f"{kind.capitalize()} artifact does not exist: {artifact_path}", + item.module_id, + )) + return record, tuple(issues) + actual_sha256 = _sha256_file(artifact_path) + record["actual_sha256"] = actual_sha256 + if actual_sha256 != expected_sha256.lower(): + issues.append(ModuleInstallerIssue( + "blocker", + "artifact_digest_mismatch", + f"{kind.capitalize()} artifact digest mismatch for {artifact_path}.", + item.module_id, + )) + return record, tuple(issues) + record["verified"] = True + record["verified_at"] = datetime.now(tz=UTC).isoformat() + return record, tuple(issues) + + +def _artifact_text(metadata: Mapping[str, object], key: str) -> str | None: + value = metadata.get(key) + if value is None: + return None + text = str(value).strip() + return text or None + + +def _artifact_path(metadata: Mapping[str, object]) -> Path | None: + value = _artifact_text(metadata, "path") or _artifact_text(metadata, "artifact_path") + if not value: + return None + if value.startswith("file://"): + value = value[len("file://"):] + path = Path(value).expanduser() + return path if path.is_absolute() else Path.cwd() / path + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def _snapshot_environment( run_dir: Path, *, @@ -1368,7 +1552,7 @@ def _snapshot_sqlite_database(run_dir: Path, database_url: str | None) -> dict[s raise ModuleInstallerError("Automatic database backup for --migrate currently supports sqlite:/// URLs only.") backup_path = run_dir / "database.sqlite.before" if db_path.exists(): - with sqlite3.connect(str(db_path)) as source, sqlite3.connect(str(backup_path)) as target: + with closing(sqlite3.connect(str(db_path))) as source, closing(sqlite3.connect(str(backup_path))) as target: source.backup(target) else: backup_path.touch() @@ -1467,7 +1651,7 @@ def _restore_database_snapshot( if backup_path.stat().st_size == 0: target_path.unlink(missing_ok=True) return {"restored": True, "type": "sqlite", "source": str(target_path), "empty": True} - with sqlite3.connect(str(backup_path)) as source, sqlite3.connect(str(target_path)) as target: + with closing(sqlite3.connect(str(backup_path))) as source, closing(sqlite3.connect(str(target_path))) as target: source.backup(target) return {"restored": True, "type": "sqlite", "source": str(target_path)} @@ -1523,6 +1707,9 @@ def _run_database_hook( }) if database_url: env["GOVOPLAN_DATABASE_URL"] = database_url + pgtools_url = _database_url_for_pgtools(database_url) + if pgtools_url: + env["GOVOPLAN_DATABASE_URL_PGTOOLS"] = pgtools_url env.setdefault("DATABASE_URL", database_url) return subprocess.run( command, @@ -1535,6 +1722,14 @@ def _run_database_hook( ) +def _database_url_for_pgtools(database_url: str) -> str | None: + if database_url.startswith(("postgresql://", "postgres://")): + return database_url + if database_url.startswith("postgresql+"): + return re.sub(r"^postgresql\+[^:]+://", "postgresql://", database_url) + return None + + def _sqlite_database_path(database_url: str | None) -> Path | None: if not database_url or not database_url.startswith("sqlite:///"): return None @@ -1572,6 +1767,7 @@ def _mark_applied(item: ModuleInstallPlanItem) -> ModuleInstallPlanItem: python_ref=item.python_ref, webui_package=item.webui_package, webui_ref=item.webui_ref, + artifact_integrity=item.artifact_integrity, destroy_data=item.destroy_data, status="applied", notes=item.notes, diff --git a/src/govoplan_core/core/module_license.py b/src/govoplan_core/core/module_license.py index c2c0940..c014b8d 100644 --- a/src/govoplan_core/core/module_license.py +++ b/src/govoplan_core/core/module_license.py @@ -11,14 +11,103 @@ import urllib.error import urllib.request from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]: + return _module_license_decision_from_validation(validate_module_license(), required_features) + + +def module_license_diagnostics( + path: Path | None = None, + *, + required_features: list[str] | tuple[str, ...] = (), + trusted_keys: dict[str, str] | None = None, + require_trusted: bool | None = None, +) -> dict[str, object]: + validation = validate_module_license(path, trusted_keys=trusted_keys, require_trusted=require_trusted) + decision = _module_license_decision_from_validation(validation, required_features) + guidance = _license_guidance(validation, decision) + return { + "configured": validation["configured"], + "valid": validation["valid"], + "path": validation["path"], + "license_id": validation["license_id"], + "subject": validation["subject"], + "features": validation["features"], + "valid_from": validation["valid_from"], + "valid_until": validation["valid_until"], + "signed": validation["signed"], + "trusted": validation["trusted"], + "key_id": validation["key_id"], + "enforced": _license_enforcement_enabled(), + "allowed": decision["allowed"], + "required_features": decision["required_features"], + "missing_features": decision["missing_features"], + "expires_in_days": _expires_in_days(validation.get("valid_until")), + "reason": decision.get("reason") or validation.get("error"), + "guidance": guidance, + } + + +def issue_module_license( + *, + path: Path, + license_id: str, + subject: str, + features: list[str] | tuple[str, ...], + valid_until: str | datetime, + signing_key_id: str, + signing_private_key_path: Path, + valid_from: str | datetime | None = None, + issuer: str | None = None, + notes: str | None = None, +) -> Path: + clean_license_id = str(license_id).strip() + clean_subject = str(subject).strip() + clean_key_id = str(signing_key_id).strip() + clean_features = _normalize_required_features(features) + if not clean_license_id: + raise ValueError("License id is required.") + if not clean_subject: + raise ValueError("License subject is required.") + if not clean_key_id: + raise ValueError("License signing key id is required.") + if not clean_features: + raise ValueError("At least one license feature is required.") + payload: dict[str, object] = { + "license_id": clean_license_id, + "subject": clean_subject, + "features": list(clean_features), + "valid_from": _datetime_text(valid_from or datetime.now(tz=UTC)), + "valid_until": _datetime_text(valid_until), + } + clean_issuer = str(issuer).strip() if issuer else "" + if clean_issuer: + payload["issuer"] = clean_issuer + clean_notes = str(notes).strip() if notes else "" + if clean_notes: + payload["notes"] = clean_notes + private_key = _load_private_key(signing_private_key_path) + signature = private_key.sign(_canonical_bytes(payload)) + payload["signature"] = { + "algorithm": "ed25519", + "key_id": clean_key_id, + "value": base64.b64encode(signature).decode("ascii"), + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def _module_license_decision_from_validation( + validation: dict[str, object], + required_features: list[str] | tuple[str, ...], +) -> dict[str, object]: required = tuple(dict.fromkeys(str(item).strip() for item in required_features if str(item).strip())) if not required: return {"allowed": True, "required_features": [], "missing_features": [], "enforced": False} - validation = validate_module_license() enforced = _license_enforcement_enabled() if not validation["valid"]: return { @@ -102,6 +191,43 @@ def _license_result( } +def _license_guidance(validation: dict[str, object], decision: dict[str, object]) -> list[str]: + guidance: list[str] = [] + if not validation.get("configured"): + guidance.append("Configure GOVOPLAN_LICENSE_FILE with the current signed license file.") + elif validation.get("path") and not validation.get("valid") and validation.get("error"): + guidance.append("Import a renewed license file or correct the configured license path before enforcing installs.") + if validation.get("configured") and not validation.get("signed"): + guidance.append("Use a signed license for production and configure GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE.") + elif validation.get("signed") and not validation.get("trusted"): + guidance.append("Add the signing public key to the trusted license keyring, or rotate to a trusted issuer key.") + expires_in_days = _expires_in_days(validation.get("valid_until")) + if expires_in_days is not None: + if expires_in_days < 0: + guidance.append("Renew the license; the validity window has ended.") + elif expires_in_days <= 30: + guidance.append(f"Renew the license soon; it expires in {expires_in_days} day(s).") + missing = decision.get("missing_features") + if isinstance(missing, list) and missing: + guidance.append("Request a renewal or entitlement update for: " + ", ".join(str(item) for item in missing)) + if not _license_enforcement_enabled(): + guidance.append("License enforcement is observe-only until GOVOPLAN_LICENSE_ENFORCEMENT=true is set.") + return guidance + + +def _expires_in_days(value: object) -> int | None: + if not value: + return None + try: + expires_at = _parse_datetime(value) + except (TypeError, ValueError): + return None + if expires_at is None: + return None + seconds = (expires_at - datetime.now(tz=UTC)).total_seconds() + return int(seconds // 86400) + + def _configured_license_path() -> Path | None: value = os.getenv("GOVOPLAN_LICENSE_FILE", "").strip() return Path(value).expanduser() if value else None @@ -203,6 +329,13 @@ def _canonical_bytes(payload: object) -> bytes: return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") +def _datetime_text(value: str | datetime) -> str: + parsed = _parse_datetime(value) + if parsed is None: + raise ValueError("License validity timestamp is required.") + return parsed.isoformat().replace("+00:00", "Z") + + def _parse_datetime(value: object) -> datetime | None: if value is None: return None @@ -223,3 +356,14 @@ def _string_list(value: object) -> list[str]: if not isinstance(value, list): raise ValueError("License features must be a list.") return [str(item).strip() for item in value if str(item).strip()] + + +def _normalize_required_features(value: list[str] | tuple[str, ...]) -> tuple[str, ...]: + return tuple(dict.fromkeys(str(item).strip() for item in value if str(item).strip())) + + +def _load_private_key(path: Path) -> Ed25519PrivateKey: + private_key = serialization.load_pem_private_key(path.expanduser().read_bytes(), password=None) + if not isinstance(private_key, Ed25519PrivateKey): + raise ValueError("License signing key must be an Ed25519 private key.") + return private_key diff --git a/src/govoplan_core/core/module_management.py b/src/govoplan_core/core/module_management.py index e34cfdd..d044e6f 100644 --- a/src/govoplan_core/core/module_management.py +++ b/src/govoplan_core/core/module_management.py @@ -42,6 +42,7 @@ class ModuleInstallPlanItem: python_ref: str | None = None webui_package: str | None = None webui_ref: str | None = None + artifact_integrity: Mapping[str, object] | None = None destroy_data: bool = False status: str = "planned" notes: str | None = None @@ -58,6 +59,8 @@ class ModuleInstallPlanItem: value = getattr(self, key) if value: payload[key] = value + if self.artifact_integrity: + payload["artifact_integrity"] = dict(self.artifact_integrity) return payload @@ -113,7 +116,7 @@ def load_startup_enabled_modules( with get_database().session() as session: desired = saved_desired_enabled_modules(session, fallback) except (RuntimeError, SQLAlchemyError): - return fallback + desired = fallback if available is None: return desired @@ -293,6 +296,7 @@ def normalize_module_install_plan_item( python_ref = _clean_optional_string(raw.get("python_ref")) webui_package = _clean_optional_string(raw.get("webui_package")) webui_ref = _clean_optional_string(raw.get("webui_ref")) + artifact_integrity = _clean_optional_mapping(raw.get("artifact_integrity"), field="artifact_integrity", module_id=module_id) destroy_data = _clean_bool(raw.get("destroy_data")) notes = _clean_optional_string(raw.get("notes")) @@ -322,6 +326,7 @@ def normalize_module_install_plan_item( python_ref=python_ref, webui_package=webui_package, webui_ref=webui_ref, + artifact_integrity=artifact_integrity, destroy_data=destroy_data, status=status, notes=notes, @@ -413,6 +418,14 @@ def _clean_bool(value: object) -> bool: return bool(value) +def _clean_optional_mapping(value: object, *, field: str, module_id: str) -> dict[str, object] | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise ModuleManagementError(f"Install plan item {module_id!r} field {field} must be an object.") + return {str(key): item for key, item in value.items() if str(key).strip()} + + def _validate_dependency_ref(value: str, *, field: str, module_id: str) -> None: lowered = value.lower() if lowered.startswith(LOCAL_DEPENDENCY_REF_PREFIXES) or value.startswith(("/", "./", "../", "~")): diff --git a/src/govoplan_core/core/module_package_catalog.py b/src/govoplan_core/core/module_package_catalog.py index dcca99e..fdb58ac 100644 --- a/src/govoplan_core/core/module_package_catalog.py +++ b/src/govoplan_core/core/module_package_catalog.py @@ -54,10 +54,13 @@ def validate_module_package_catalog( "path": str(catalog_source), "source": str(catalog_source), "source_type": _catalog_source_type(catalog_source), + "cache_used": False, + "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None, "modules": [], "channel": None, "sequence": None, "generated_at": None, + "not_before": None, "expires_at": None, "signed": False, "trusted": False, @@ -66,12 +69,14 @@ def validate_module_package_catalog( "error": f"Module package catalog does not exist: {catalog_source}", } warnings: list[str] = [] + read_state = {"cache_used": False, "cache_path": str(_configured_catalog_cache_path()) if _configured_catalog_cache_path() is not None else None} try: - payload = _read_catalog_payload(catalog_source) + payload, read_state = _read_catalog_payload_with_metadata(catalog_source) modules = _normalize_catalog_modules(payload) channel = _catalog_channel(payload) sequence = _catalog_sequence(payload) generated_at = _catalog_optional_text(payload, "generated_at") + not_before = _catalog_optional_text(payload, "not_before") expires_at = _catalog_optional_text(payload, "expires_at") signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys) freshness = _catalog_freshness_state(payload) @@ -83,10 +88,13 @@ def validate_module_package_catalog( "path": str(catalog_source) if catalog_source is not None else None, "source": str(catalog_source) if catalog_source is not None else None, "source_type": _catalog_source_type(catalog_source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, "modules": [], "channel": None, "sequence": None, "generated_at": None, + "not_before": None, "expires_at": None, "signed": False, "trusted": False, @@ -101,10 +109,13 @@ def validate_module_package_catalog( "path": str(catalog_source) if catalog_source is not None else None, "source": str(catalog_source) if catalog_source is not None else None, "source_type": _catalog_source_type(catalog_source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, "modules": [], "channel": channel, "sequence": sequence, "generated_at": generated_at, + "not_before": not_before, "expires_at": expires_at, "signed": signature_state["signed"], "trusted": signature_state["trusted"], @@ -119,10 +130,13 @@ def validate_module_package_catalog( "path": str(catalog_source) if catalog_source is not None else None, "source": str(catalog_source) if catalog_source is not None else None, "source_type": _catalog_source_type(catalog_source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, "modules": [], "channel": channel, "sequence": sequence, "generated_at": generated_at, + "not_before": not_before, "expires_at": expires_at, "signed": signature_state["signed"], "trusted": signature_state["trusted"], @@ -137,10 +151,13 @@ def validate_module_package_catalog( "path": str(catalog_source) if catalog_source is not None else None, "source": str(catalog_source) if catalog_source is not None else None, "source_type": _catalog_source_type(catalog_source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, "modules": [], "channel": channel, "sequence": sequence, "generated_at": generated_at, + "not_before": not_before, "expires_at": expires_at, "signed": signature_state["signed"], "trusted": False, @@ -155,8 +172,10 @@ def validate_module_package_catalog( channel=channel, sequence=sequence, generated_at=generated_at, + not_before=not_before, expires_at=expires_at, signature_state=signature_state, + read_state=read_state, error=str(freshness["error"]), ) if not replay["valid"]: @@ -166,8 +185,10 @@ def validate_module_package_catalog( channel=channel, sequence=sequence, generated_at=generated_at, + not_before=not_before, expires_at=expires_at, signature_state=signature_state, + read_state=read_state, error=str(replay["error"]), ) warnings.extend(str(item) for item in freshness.get("warnings", ()) if item) @@ -182,10 +203,13 @@ def validate_module_package_catalog( "path": str(catalog_source) if catalog_source is not None else None, "source": str(catalog_source) if catalog_source is not None else None, "source_type": _catalog_source_type(catalog_source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, "modules": list(modules), "channel": channel, "sequence": sequence, "generated_at": generated_at, + "not_before": not_before, "expires_at": expires_at, "signed": signature_state["signed"], "trusted": signature_state["trusted"], @@ -353,14 +377,31 @@ def _parse_trusted_keys(value: str) -> dict[str, str]: def _read_catalog_payload(source: Path | str | None) -> object: + payload, _metadata = _read_catalog_payload_with_metadata(source) + return payload + + +def _read_catalog_payload_with_metadata(source: Path | str | None) -> tuple[object, dict[str, object]]: + cache_path = _configured_catalog_cache_path() + metadata: dict[str, object] = { + "cache_used": False, + "cache_path": str(cache_path) if cache_path is not None else None, + } if source is None: - return {"modules": []} + return {"modules": []}, metadata if isinstance(source, str) and _is_http_url(source): - return json.loads(_read_catalog_url(source)) - return json.loads(Path(source).read_text(encoding="utf-8")) + body, cache_used = _read_catalog_url_with_metadata(source) + metadata["cache_used"] = cache_used + return json.loads(body), metadata + return json.loads(Path(source).read_text(encoding="utf-8")), metadata def _read_catalog_url(url: str) -> str: + body, _cache_used = _read_catalog_url_with_metadata(url) + return body + + +def _read_catalog_url_with_metadata(url: str) -> tuple[str, bool]: cache_path = _configured_catalog_cache_path() try: with urllib.request.urlopen(url, timeout=15) as response: @@ -368,10 +409,10 @@ def _read_catalog_url(url: str) -> str: if cache_path is not None: cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.write_text(body, encoding="utf-8") - return body + return body, False except (OSError, urllib.error.URLError): if cache_path is not None and cache_path.exists(): - return cache_path.read_text(encoding="utf-8") + return cache_path.read_text(encoding="utf-8"), True raise @@ -549,7 +590,7 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]: action = str(value.get("action") or "install") if action not in {"install", "uninstall"}: raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}") - return { + item = { "module_id": module_id, "name": str(value.get("name") or module_id), "description": _optional_str(value, "description"), @@ -563,6 +604,10 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]: "notes": _optional_str(value, "notes"), "tags": _string_list(value.get("tags")), } + artifact_integrity = _normalize_artifact_integrity(value.get("artifact_integrity")) + if artifact_integrity: + item["artifact_integrity"] = artifact_integrity + return item def _required_str(value: dict[str, Any], key: str) -> str: @@ -588,6 +633,37 @@ def _string_list(value: Any) -> list[str]: return [str(item).strip() for item in value if str(item).strip()] +def _normalize_artifact_integrity(value: Any) -> dict[str, object]: + if value is None: + return {} + if not isinstance(value, dict): + raise ValueError("Module package catalog artifact_integrity must be an object.") + normalized: dict[str, object] = {} + for key in ("python", "webui"): + raw = value.get(key) + if raw is None: + continue + if not isinstance(raw, dict): + raise ValueError(f"Module package catalog artifact_integrity.{key} must be an object.") + clean = { + field: text + for field in ( + "ref", + "path", + "artifact_path", + "sha256", + "sbom_url", + "provenance_url", + "registry_identity", + "git_ref", + ) + if (text := _optional_str(raw, field)) + } + if clean: + normalized[key] = clean + return normalized + + def _parse_datetime(value: object) -> datetime | None: if value is None: return None @@ -628,8 +704,10 @@ def _invalid_catalog_result( channel: str | None, sequence: int | None, generated_at: str | None, + not_before: str | None, expires_at: str | None, signature_state: dict[str, object], + read_state: dict[str, object], error: str, ) -> dict[str, object]: return { @@ -638,10 +716,13 @@ def _invalid_catalog_result( "path": str(source) if source is not None else None, "source": str(source) if source is not None else None, "source_type": _catalog_source_type(source), + "cache_used": bool(read_state.get("cache_used")), + "cache_path": read_state.get("cache_path") if isinstance(read_state.get("cache_path"), str) else None, "modules": list(modules), "channel": channel, "sequence": sequence, "generated_at": generated_at, + "not_before": not_before, "expires_at": expires_at, "signed": signature_state["signed"], "trusted": signature_state["trusted"], diff --git a/src/govoplan_core/core/modules.py b/src/govoplan_core/core/modules.py index c16324e..317f573 100644 --- a/src/govoplan_core/core/modules.py +++ b/src/govoplan_core/core/modules.py @@ -8,6 +8,9 @@ if TYPE_CHECKING: from fastapi import APIRouter +SUPPORTED_MANIFEST_CONTRACT_VERSION = "1" +SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION = "1" + PermissionLevel = Literal["system", "tenant"] SubjectType = Literal["account", "membership", "group", "service_account", "tenant"] @@ -129,6 +132,61 @@ class ModuleContext: data: Mapping[str, Any] = field(default_factory=dict) +DocumentationLayer = Literal["always", "configured", "available", "evidence"] +DocumentationLinkKind = Literal["runtime", "api", "repository", "wiki", "public"] +DocumentationType = Literal["admin", "user"] + + +@dataclass(frozen=True, slots=True) +class DocumentationLink: + label: str + href: str + kind: DocumentationLinkKind = "runtime" + + +@dataclass(frozen=True, slots=True) +class DocumentationCondition: + required_modules: tuple[str, ...] = () + any_modules: tuple[str, ...] = () + missing_modules: tuple[str, ...] = () + required_capabilities: tuple[str, ...] = () + required_scopes: tuple[str, ...] = () + any_scopes: tuple[str, ...] = () + configuration_keys: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class DocumentationTopic: + id: str + title: str + summary: str + body: str = "" + layer: DocumentationLayer = "configured" + documentation_types: tuple[DocumentationType, ...] = ("admin",) + audience: tuple[str, ...] = () + order: int = 100 + conditions: tuple[DocumentationCondition, ...] = () + links: tuple[DocumentationLink, ...] = () + related_modules: tuple[str, ...] = () + unlocks: tuple[str, ...] = () + configuration_keys: tuple[str, ...] = () + i18n_key: str | None = None + translations: Mapping[str, Mapping[str, str]] = field(default_factory=dict) + source_module_id: str | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class DocumentationContext: + registry: object + principal: object | None = None + settings: object | None = None + session: object | None = None + documentation_type: DocumentationType = "admin" + locale: str = "en" + data: Mapping[str, Any] = field(default_factory=dict) + + class ResourceAclProvider(Protocol): resource_type: str @@ -146,6 +204,7 @@ TenantSummaryProvider = Callable[[object, str], Mapping[str, int]] DeleteVetoProvider = Callable[[object, str, str], None] RouteFactory = Callable[[ModuleContext], "APIRouter"] CapabilityFactory = Callable[[ModuleContext], object] +DocumentationProvider = Callable[[DocumentationContext], Iterable[DocumentationTopic]] LifecycleHook = Callable[[ModuleContext], None] @@ -170,3 +229,5 @@ class ModuleManifest: compatibility: ModuleCompatibility = field(default_factory=ModuleCompatibility) on_activate: LifecycleHook | None = None on_deactivate: LifecycleHook | None = None + documentation: tuple[DocumentationTopic, ...] = () + documentation_providers: tuple[DocumentationProvider, ...] = () diff --git a/src/govoplan_core/core/organizations.py b/src/govoplan_core/core/organizations.py new file mode 100644 index 0000000..e000149 --- /dev/null +++ b/src/govoplan_core/core/organizations.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Literal, Protocol, runtime_checkable + + +ORGANIZATIONS_MODULE_ID = "organizations" +CAPABILITY_ORGANIZATION_DIRECTORY = "organizations.directory" + +OrganizationStatus = Literal["active", "inactive", "suspended"] +FunctionAssignmentSource = Literal["direct", "delegated", "acting_for", "directory", "governance", "system"] + + +@dataclass(frozen=True, slots=True) +class OrganizationUnitRef: + id: str + tenant_id: str + slug: str + name: str + parent_id: str | None = None + description: str | None = None + status: OrganizationStatus = "active" + + +@dataclass(frozen=True, slots=True) +class OrganizationFunctionRef: + id: str + tenant_id: str + organization_unit_id: str + slug: str + name: str + description: str | None = None + delegable: bool = False + act_in_place_allowed: bool = False + status: OrganizationStatus = "active" + + +@dataclass(frozen=True, slots=True) +class OrganizationFunctionAssignmentRef: + id: str + tenant_id: str + account_id: str + function_id: str + organization_unit_id: str + identity_id: str | None = None + applies_to_subunits: bool = False + source: FunctionAssignmentSource = "direct" + delegated_from_assignment_id: str | None = None + acting_for_account_id: str | None = None + valid_from: datetime | None = None + valid_until: datetime | None = None + status: OrganizationStatus = "active" + + +@runtime_checkable +class OrganizationDirectory(Protocol): + def get_organization_unit(self, organization_unit_id: str) -> OrganizationUnitRef | None: + ... + + def organization_units_for_tenant(self, tenant_id: str) -> Sequence[OrganizationUnitRef]: + ... + + def get_function(self, function_id: str) -> OrganizationFunctionRef | None: + ... + + def functions_for_organization_unit( + self, + organization_unit_id: str, + *, + include_subunits: bool = False, + ) -> Sequence[OrganizationFunctionRef]: + ... + + def get_function_assignment(self, assignment_id: str) -> OrganizationFunctionAssignmentRef | None: + ... + + def function_assignments_for_account( + self, + account_id: str, + *, + tenant_id: str | None = None, + ) -> Sequence[OrganizationFunctionAssignmentRef]: + ... diff --git a/src/govoplan_core/core/pagination.py b/src/govoplan_core/core/pagination.py new file mode 100644 index 0000000..5ff4a74 --- /dev/null +++ b/src/govoplan_core/core/pagination.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from datetime import date, datetime +from decimal import Decimal +from typing import Any, Mapping + +KEYSET_CURSOR_PREFIX = "ks1:" + + +class KeysetCursorError(ValueError): + """Raised when a keyset cursor cannot be decoded for the current query.""" + + +def keyset_query_fingerprint(scope: str, params: Mapping[str, Any]) -> str: + payload = {"scope": scope, "params": _jsonable(params)} + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] + + +def encode_keyset_cursor(scope: str, *, fingerprint: str, values: Mapping[str, Any]) -> str: + payload = { + "v": 1, + "scope": scope, + "fingerprint": fingerprint, + "values": _jsonable(values), + } + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + encoded = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + return f"{KEYSET_CURSOR_PREFIX}{encoded}" + + +def decode_keyset_cursor(scope: str, cursor: str | None, *, fingerprint: str | None = None) -> dict[str, Any] | None: + if cursor is None or not cursor.strip(): + return None + raw_cursor = cursor.strip() + if not raw_cursor.startswith(KEYSET_CURSOR_PREFIX): + raise KeysetCursorError("Invalid pagination cursor") + encoded = raw_cursor[len(KEYSET_CURSOR_PREFIX):] + try: + padded = encoded + ("=" * (-len(encoded) % 4)) + payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")) + except (UnicodeDecodeError, ValueError, json.JSONDecodeError) as exc: + raise KeysetCursorError("Invalid pagination cursor") from exc + if not isinstance(payload, dict) or payload.get("v") != 1: + raise KeysetCursorError("Invalid pagination cursor") + if payload.get("scope") != scope: + raise KeysetCursorError("Pagination cursor does not match this endpoint") + if fingerprint is not None and payload.get("fingerprint") != fingerprint: + raise KeysetCursorError("Pagination cursor does not match the current query") + values = payload.get("values") + if not isinstance(values, dict): + raise KeysetCursorError("Invalid pagination cursor") + return values + + +def _jsonable(value: Any) -> Any: + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, Mapping): + return {str(key): _jsonable(item) for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + return value diff --git a/src/govoplan_core/core/policy.py b/src/govoplan_core/core/policy.py new file mode 100644 index 0000000..1129a7e --- /dev/null +++ b/src/govoplan_core/core/policy.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Iterable, Literal, Mapping, cast +from urllib.parse import quote, unquote + +PolicyScopeType = Literal["system", "tenant", "user", "group", "campaign"] + +POLICY_SCOPE_TYPES: tuple[PolicyScopeType, ...] = ("system", "tenant", "user", "group", "campaign") + + +def normalize_policy_scope_type(scope_type: str) -> PolicyScopeType: + clean = scope_type.strip().casefold() + if clean not in POLICY_SCOPE_TYPES: + raise ValueError("Policy scope must be system, tenant, user, group or campaign") + return cast(PolicyScopeType, clean) + + +@dataclass(frozen=True, slots=True) +class PolicySourceRef: + scope_type: PolicyScopeType + scope_id: str | None = None + + @property + def path(self) -> str: + return policy_source_path(self.scope_type, self.scope_id) + + def to_dict(self) -> dict[str, Any]: + return {"scope_type": self.scope_type, "scope_id": self.scope_id, "path": self.path} + + +def policy_source_path(scope_type: str, scope_id: str | None = None) -> str: + clean_scope = normalize_policy_scope_type(scope_type) + if clean_scope == "system": + if scope_id: + raise ValueError("System policy sources do not carry a scope_id") + return "system" + if not scope_id: + raise ValueError(f"{clean_scope.capitalize()} policy sources require a scope_id") + return f"{clean_scope}:{quote(str(scope_id), safe='')}" + + +def parse_policy_source_path(path: str) -> PolicySourceRef: + clean_path = path.strip() + if clean_path == "system": + return PolicySourceRef(scope_type="system") + scope_type, separator, encoded_scope_id = clean_path.partition(":") + if not separator: + raise ValueError("Policy source path must be system or :") + clean_scope = normalize_policy_scope_type(scope_type) + if clean_scope == "system": + raise ValueError("System policy source path must be exactly system") + scope_id = unquote(encoded_scope_id) + if not scope_id: + raise ValueError("Policy source path requires a non-empty scope id") + return PolicySourceRef(scope_type=clean_scope, scope_id=scope_id) + + +@dataclass(frozen=True, slots=True) +class PolicySourceStep: + scope_type: PolicyScopeType + label: str + scope_id: str | None = None + applied_fields: tuple[str, ...] = () + policy: Mapping[str, Any] = field(default_factory=dict) + + @property + def path(self) -> str: + return policy_source_path(self.scope_type, self.scope_id) + + def to_dict(self) -> dict[str, Any]: + return { + "scope_type": self.scope_type, + "scope_id": self.scope_id, + "path": self.path, + "label": self.label, + "applied_fields": list(self.applied_fields), + "policy": dict(self.policy), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> PolicySourceStep: + policy_value = value.get("policy") + return cls( + scope_type=normalize_policy_scope_type(str(value.get("scope_type", ""))), + scope_id=str(value["scope_id"]) if value.get("scope_id") is not None else None, + label=str(value.get("label") or ""), + applied_fields=tuple(str(field) for field in (value.get("applied_fields") or ())), + policy=policy_value if isinstance(policy_value, Mapping) else {}, + ) + + +def policy_source_step( + scope_type: str, + label: str, + scope_id: str | None, + applied_fields: Iterable[str] = (), + policy: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + return PolicySourceStep( + scope_type=normalize_policy_scope_type(scope_type), + scope_id=scope_id, + label=label, + applied_fields=tuple(str(field) for field in applied_fields), + policy=dict(policy or {}), + ).to_dict() + + +@dataclass(frozen=True, slots=True) +class PolicyDecision: + allowed: bool + reason: str | None = None + source_path: tuple[PolicySourceStep, ...] = () + requirements: tuple[str, ...] = () + details: Mapping[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "allowed": self.allowed, + "reason": self.reason, + "source_path": [step.to_dict() for step in self.source_path], + "requirements": list(self.requirements), + "details": dict(self.details), + } diff --git a/src/govoplan_core/core/registry.py b/src/govoplan_core/core/registry.py index 16e3857..280e512 100644 --- a/src/govoplan_core/core/registry.py +++ b/src/govoplan_core/core/registry.py @@ -14,9 +14,13 @@ from govoplan_core.core.modules import ( PermissionDefinition, ResourceAclProvider, RoleTemplate, + SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION, + SUPPORTED_MANIFEST_CONTRACT_VERSION, TenantSummaryProvider, ) +_MODULE_ID_RE = re.compile(r"^[a-z][a-z0-9_]*$") +_NPM_PACKAGE_RE = re.compile(r"^(?:@[a-z0-9][a-z0-9_.-]*/)?[a-z0-9][a-z0-9_.-]*$") _SCOPE_RE = re.compile(r"^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$") _WILDCARD_RE = re.compile(r"^([a-z][a-z0-9_]*|\*):\*$|^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*:\*$") @@ -150,6 +154,7 @@ class PlatformRegistry: ordered = tuple(self._topologically_sorted()) seen_permissions: dict[str, PermissionDefinition] = {} for manifest in ordered: + _validate_manifest_shape(manifest) for dependency in manifest.dependencies: if dependency not in self._manifests: raise RegistryError(f"Module {manifest.id!r} depends on unknown module {dependency!r}") @@ -207,3 +212,73 @@ class PlatformRegistry: unresolved = ", ".join(sorted(module_id for module_id, dependencies in incoming.items() if dependencies)) raise RegistryError(f"Module dependency cycle or unresolved dependency: {unresolved}") return (self._manifests[module_id] for module_id in ordered) + + +def _validate_manifest_shape(manifest: ModuleManifest) -> None: + if not _MODULE_ID_RE.match(manifest.id): + raise RegistryError(f"Module manifest id must match {_MODULE_ID_RE.pattern}: {manifest.id!r}") + if not manifest.name.strip(): + raise RegistryError(f"Module {manifest.id!r} must declare a non-empty name") + if not manifest.version.strip(): + raise RegistryError(f"Module {manifest.id!r} must declare a non-empty version") + if manifest.compatibility.manifest_contract_version != SUPPORTED_MANIFEST_CONTRACT_VERSION: + raise RegistryError( + f"Module {manifest.id!r} uses unsupported manifest contract version " + f"{manifest.compatibility.manifest_contract_version!r}; supported version is " + f"{SUPPORTED_MANIFEST_CONTRACT_VERSION!r}" + ) + + _validate_dependency_list(manifest.id, "dependencies", manifest.dependencies) + _validate_dependency_list(manifest.id, "optional_dependencies", manifest.optional_dependencies) + overlap = set(manifest.dependencies) & set(manifest.optional_dependencies) + if overlap: + joined = ", ".join(sorted(overlap)) + raise RegistryError(f"Module {manifest.id!r} lists dependencies as both required and optional: {joined}") + + if manifest.migration_spec is not None: + if manifest.migration_spec.module_id != manifest.id: + raise RegistryError(f"Module {manifest.id!r} has migration spec for {manifest.migration_spec.module_id!r}") + if manifest.migration_spec.metadata is None and not manifest.migration_spec.script_location: + raise RegistryError(f"Module {manifest.id!r} migration spec must declare metadata or script location") + + if manifest.frontend is not None: + frontend = manifest.frontend + if frontend.module_id != manifest.id: + raise RegistryError(f"Module {manifest.id!r} has frontend metadata for {frontend.module_id!r}") + if frontend.asset_manifest_contract_version != SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION: + raise RegistryError( + f"Module {manifest.id!r} uses unsupported frontend asset manifest contract version " + f"{frontend.asset_manifest_contract_version!r}; supported version is " + f"{SUPPORTED_FRONTEND_ASSET_MANIFEST_CONTRACT_VERSION!r}" + ) + if frontend.package_name is not None and not _NPM_PACKAGE_RE.match(frontend.package_name): + raise RegistryError(f"Module {manifest.id!r} has invalid frontend package name {frontend.package_name!r}") + for route in (*frontend.routes, *frontend.settings_routes): + if not route.path.startswith("/"): + raise RegistryError(f"Frontend route for module {manifest.id!r} must start with '/': {route.path!r}") + if not route.component.strip(): + raise RegistryError(f"Frontend route {route.path!r} for module {manifest.id!r} must declare a component") + for item in frontend.nav_items: + _validate_nav_item(manifest.id, item) + + for item in manifest.nav_items: + _validate_nav_item(manifest.id, item) + + +def _validate_dependency_list(module_id: str, field_name: str, dependencies: tuple[str, ...]) -> None: + if len(dependencies) != len(set(dependencies)): + raise RegistryError(f"Module {module_id!r} has duplicate {field_name}") + for dependency in dependencies: + if dependency == module_id: + raise RegistryError(f"Module {module_id!r} cannot depend on itself") + if not _MODULE_ID_RE.match(dependency): + raise RegistryError(f"Module {module_id!r} has invalid dependency id {dependency!r}") + + +def _validate_nav_item(module_id: str, item: NavItem) -> None: + if not item.path.startswith("/"): + raise RegistryError(f"Navigation item for module {module_id!r} must start with '/': {item.path!r}") + if not item.label.strip(): + raise RegistryError(f"Navigation item {item.path!r} for module {module_id!r} must declare a label") + if item.icon is not None and not item.icon.strip(): + raise RegistryError(f"Navigation item {item.path!r} for module {module_id!r} has an empty icon name") diff --git a/src/govoplan_core/db/bootstrap.py b/src/govoplan_core/db/bootstrap.py index 6359e4d..ecd9606 100644 --- a/src/govoplan_core/db/bootstrap.py +++ b/src/govoplan_core/db/bootstrap.py @@ -29,6 +29,7 @@ def create_all_tables() -> None: # Build the configured registry so enabled module manifests register their # model metadata with the shared SQLAlchemy base before create_all runs. from govoplan_core.admin import models as core_admin_models # noqa: F401 + from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules) candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules) diff --git a/src/govoplan_core/db/migrations.py b/src/govoplan_core/db/migrations.py index 45dc6b0..58f532e 100644 --- a/src/govoplan_core/db/migrations.py +++ b/src/govoplan_core/db/migrations.py @@ -10,6 +10,8 @@ from alembic.runtime.migration import MigrationContext from sqlalchemy import create_engine, inspect, text from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan +from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata +from govoplan_core.core.change_sequence import ChangeSequenceEntry, ChangeSequenceRetentionFloor from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids from govoplan_core.db.session import configure_database from govoplan_core.server.default_config import get_server_config @@ -26,8 +28,27 @@ from govoplan_core.settings import settings REVISION_AUTH_RBAC = "2c3d4e5f6a7b" REVISION_FILE_STORAGE = "3d4e5f6a7b8c" REVISION_FILE_FOLDERS = "4e5f6a7b8c9d" +REVISION_NAMESPACE_PLATFORM_TABLES = "2e3f4a5b6c7d" +REVISION_CORE_CHANGE_SEQUENCE = "3f4a5b6c7d8e" REVISION_HIERARCHICAL_SETTINGS = "f5a6b7c8d9e0" +_NAMESPACE_TABLE_RENAMES = ( + ("tenants", "tenancy_tenants"), + ("accounts", "access_accounts"), + ("users", "access_users"), + ("groups", "access_groups"), + ("roles", "access_roles"), + ("system_role_assignments", "access_system_role_assignments"), + ("user_group_memberships", "access_user_group_memberships"), + ("user_role_assignments", "access_user_role_assignments"), + ("group_role_assignments", "access_group_role_assignments"), + ("api_keys", "access_api_keys"), + ("auth_sessions", "access_auth_sessions"), + ("system_settings", "core_system_settings"), + ("governance_templates", "admin_governance_templates"), + ("governance_template_assignments", "admin_governance_template_assignments"), +) + _FILE_STORAGE_TABLES = { "file_blobs", "file_assets", @@ -74,21 +95,21 @@ _FILE_STORAGE_COLUMNS = { } _CREATE_ALL_THROUGH_HIERARCHICAL_TABLES = { - "accounts", - "auth_sessions", + "access_accounts", + "access_auth_sessions", "audit_log", "campaign_versions", "campaign_jobs", "send_attempts", - "system_role_assignments", - "system_settings", - "governance_templates", - "governance_template_assignments", + "access_system_role_assignments", + "core_system_settings", + "admin_governance_templates", + "admin_governance_template_assignments", "mail_server_profiles", } _CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS = { - "auth_sessions": {"account_id", "csrf_token_hash"}, + "access_auth_sessions": {"account_id", "csrf_token_hash"}, "audit_log": {"scope", "tenant_id", "user_id", "api_key_id"}, "campaign_versions": { "workflow_state", @@ -102,13 +123,13 @@ _CREATE_ALL_THROUGH_HIERARCHICAL_COLUMNS = { }, "campaign_jobs": {"claimed_at", "claim_token", "smtp_started_at", "outcome_unknown_at", "eml_sha256"}, "send_attempts": {"status", "claim_token"}, - "users": {"account_id", "settings", "mail_profile_policy"}, - "groups": {"system_template_id", "system_required", "settings", "mail_profile_policy"}, - "roles": {"system_template_id", "system_required"}, - "tenants": {"settings", "allow_custom_groups", "allow_custom_roles", "allow_api_keys"}, + "access_users": {"account_id", "settings", "mail_profile_policy"}, + "access_groups": {"system_template_id", "system_required", "settings", "mail_profile_policy"}, + "access_roles": {"system_template_id", "system_required"}, + "tenancy_tenants": {"settings", "allow_custom_groups", "allow_custom_roles", "allow_api_keys"}, "campaigns": {"settings", "mail_profile_policy"}, "mail_server_profiles": {"scope_type", "scope_id"}, - "system_settings": {"settings", "allow_tenant_custom_groups", "allow_tenant_custom_roles", "allow_tenant_api_keys"}, + "core_system_settings": {"settings", "allow_tenant_custom_groups", "allow_tenant_custom_roles", "allow_tenant_api_keys"}, } _CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = { @@ -121,10 +142,10 @@ _CREATE_ALL_THROUGH_HIERARCHICAL_INDEXES = { "campaign_jobs": {"ix_campaign_jobs_claim_token", "ix_campaign_jobs_eml_sha256"}, "send_attempts": {"ix_send_attempts_status", "ix_send_attempts_claim_token"}, "audit_log": {"ix_audit_log_scope_created_at", "ix_audit_log_tenant_scope_created_at"}, - "auth_sessions": {"ix_auth_sessions_account_id"}, - "users": {"ix_users_account_id"}, - "groups": {"ix_groups_system_template_id"}, - "roles": {"ix_roles_system_template_id"}, + "access_auth_sessions": {"ix_access_auth_sessions_account_id"}, + "access_users": {"ix_access_users_account_id"}, + "access_groups": {"ix_access_groups_system_template_id"}, + "access_roles": {"ix_access_roles_system_template_id"}, "mail_server_profiles": {"ix_mail_server_profiles_scope_type", "ix_mail_server_profiles_scope_id", "ix_mail_server_profiles_scope"}, } @@ -258,6 +279,111 @@ def _backfill_user_lock_state_for_create_all_schema(database_url: str) -> None: engine.dispose() +def _row_count(connection, table_name: str) -> int: + quoted = connection.dialect.identifier_preparer.quote(table_name) + return int(connection.execute(text(f"SELECT COUNT(*) FROM {quoted}")).scalar_one()) + + +def _drop_table(connection, table_name: str) -> None: + quoted = connection.dialect.identifier_preparer.quote(table_name) + connection.execute(text(f"DROP TABLE {quoted}")) + + +def _rename_table(connection, old_name: str, new_name: str) -> None: + quoted_old = connection.dialect.identifier_preparer.quote(old_name) + quoted_new = connection.dialect.identifier_preparer.quote(new_name) + connection.execute(text(f"ALTER TABLE {quoted_old} RENAME TO {quoted_new}")) + + +def reconcile_namespace_table_drift(database_url: str | None = None) -> bool: + """Repair dev databases stamped past the namespace-table migration. + + During the repository split some development databases were stamped at the + newer Alembic heads while still carrying the old platform table names. The + real migration only renames tables, so replay that idempotent operation + before startup bootstrap code touches the ORM. + """ + + url = database_url or settings.database_url + changed = False + engine = create_engine(url) + try: + with engine.begin() as connection: + schema = inspect(connection) + tables = set(schema.get_table_names()) + if not any(old_name in tables and new_name not in tables for old_name, new_name in _NAMESPACE_TABLE_RENAMES): + return False + + old_tables_to_drop: set[str] = set() + new_tables_to_drop: set[str] = set() + for old_name, new_name in _NAMESPACE_TABLE_RENAMES: + if old_name not in tables or new_name not in tables: + continue + if _row_count(connection, old_name) == 0: + old_tables_to_drop.add(old_name) + elif _row_count(connection, new_name) == 0: + new_tables_to_drop.add(new_name) + else: + raise RuntimeError(f"Cannot reconcile non-empty {old_name} over non-empty {new_name}") + + for old_name, _new_name in reversed(_NAMESPACE_TABLE_RENAMES): + if old_name in old_tables_to_drop: + _drop_table(connection, old_name) + tables.remove(old_name) + changed = True + for _old_name, new_name in reversed(_NAMESPACE_TABLE_RENAMES): + if new_name in new_tables_to_drop: + _drop_table(connection, new_name) + tables.remove(new_name) + changed = True + + for old_name, new_name in _NAMESPACE_TABLE_RENAMES: + if old_name not in tables or new_name in tables: + continue + _rename_table(connection, old_name, new_name) + tables.remove(old_name) + tables.add(new_name) + changed = True + finally: + engine.dispose() + return changed + + +def reconcile_change_sequence_retention_floor_drift(database_url: str | None = None) -> bool: + """Repair databases stamped after the change-sequence migration changed. + + Early development databases may have applied the change-sequence revision + before the retention-floor table was added to that migration. The main + change-sequence table is enough for full snapshots, but incremental delta + requests need the retention floor to decide whether a watermark is stale. + """ + + url = database_url or settings.database_url + changed = False + engine = create_engine(url) + try: + with engine.begin() as connection: + schema = inspect(connection) + tables = set(schema.get_table_names()) + if ChangeSequenceEntry.__tablename__ not in tables: + return False + if ChangeSequenceRetentionFloor.__tablename__ not in tables: + ChangeSequenceRetentionFloor.__table__.create(bind=connection, checkfirst=True) + changed = True + else: + indexes = { + index["name"] + for index in schema.get_indexes(ChangeSequenceRetentionFloor.__tablename__) + } + for index in ChangeSequenceRetentionFloor.__table__.indexes: + if index.name not in indexes: + index.create(bind=connection) + changed = True + finally: + engine.dispose() + return changed + + def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | None: """Repair the known Alembic/create_all drift without modifying application data. @@ -327,6 +453,9 @@ def migrate_database( manifest_factories: tuple[ManifestFactory, ...] = (), ) -> MigrationResult: url = database_url or settings.database_url + if reconcile_legacy_schema: + reconcile_namespace_table_drift(url) + reconcile_change_sequence_retention_floor_drift(url) previous = database_revision(url) reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None command.upgrade( diff --git a/src/govoplan_core/db/session.py b/src/govoplan_core/db/session.py index 04217c5..5912824 100644 --- a/src/govoplan_core/db/session.py +++ b/src/govoplan_core/db/session.py @@ -38,15 +38,21 @@ class DatabaseHandle: with self.SessionLocal() as session: yield session + def dispose(self) -> None: + self.engine.dispose() + _default_database: DatabaseHandle | None = None -def configure_database(database_url: str, *, engine: Engine | None = None) -> DatabaseHandle: +def configure_database(database_url: str, *, engine: Engine | None = None, dispose_previous: bool = False) -> DatabaseHandle: global _default_database if engine is None and _default_database is not None and _default_database.database_url == database_url: return _default_database + previous_database = _default_database _default_database = DatabaseHandle(database_url, engine=engine) + if dispose_previous and previous_database is not None and previous_database is not _default_database: + previous_database.dispose() return _default_database @@ -56,6 +62,13 @@ def set_database(handle: DatabaseHandle) -> DatabaseHandle: return handle +def reset_database(*, dispose: bool = False) -> None: + global _default_database + if dispose and _default_database is not None: + _default_database.dispose() + _default_database = None + + def get_database() -> DatabaseHandle: if _default_database is None: raise RuntimeError("GovOPlaN database is not configured") diff --git a/src/govoplan_core/devserver.py b/src/govoplan_core/devserver.py index ba8314d..ed77c8d 100644 --- a/src/govoplan_core/devserver.py +++ b/src/govoplan_core/devserver.py @@ -20,6 +20,9 @@ from govoplan_core.server.registry import available_module_manifests, build_plat DEFAULT_APP = "govoplan_core.server.app:app" DEFAULT_CONFIG = "govoplan_core.server.default_config:get_server_config" +DEFAULT_DEV_DATABASE_URL = "postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev" +DEFAULT_DEV_DATABASE_URL_PGTOOLS = "postgresql://govoplan_dev@127.0.0.1:5432/govoplan_dev" +DEFAULT_SQLITE_DATABASE_NAME = "multimailer-dev.db" @dataclass(slots=True) @@ -61,13 +64,23 @@ def apply_runtime_defaults(config_path: str | None) -> Path | None: (runtime_root / "runtime" / "files").mkdir(parents=True, exist_ok=True) (runtime_root / "runtime" / "mock-mailbox").mkdir(parents=True, exist_ok=True) + dev_database_backend = os.getenv("GOVOPLAN_DEV_DATABASE_BACKEND", "postgres").strip().lower() + default_database_url = DEFAULT_DEV_DATABASE_URL + default_pgtools_url = DEFAULT_DEV_DATABASE_URL_PGTOOLS + if dev_database_backend == "sqlite": + default_database_url = f"sqlite:///{runtime_root / 'runtime' / DEFAULT_SQLITE_DATABASE_NAME}" + default_pgtools_url = "" + defaults = { - "DATABASE_URL": f"sqlite:///{runtime_root / 'runtime' / 'multimailer-dev.db'}", + "DATABASE_URL": os.getenv("GOVOPLAN_DEV_DATABASE_URL", default_database_url), + "DEV_BOOTSTRAP_ENABLED": "true", "FILE_STORAGE_LOCAL_ROOT": str(runtime_root / "runtime" / "files"), "MOCK_MAILBOX_DIR": str(runtime_root / "runtime" / "mock-mailbox"), } for key, value in defaults.items(): os.environ.setdefault(key, value) + if default_pgtools_url: + os.environ.setdefault("GOVOPLAN_DATABASE_URL_PGTOOLS", os.getenv("GOVOPLAN_DEV_DATABASE_URL_PGTOOLS", default_pgtools_url)) return runtime_root @@ -95,7 +108,7 @@ def validate_sqlite_database_url(database_url: str) -> None: raise SystemExit( f"DATABASE_URL points to {db_path}, but its parent directory does not exist. " "Unset stale DATABASE_URL or set it to " - f"sqlite:///{Path.cwd().resolve() / 'runtime' / 'multimailer-dev.db'}." + f"sqlite:///{Path.cwd().resolve() / 'runtime' / DEFAULT_SQLITE_DATABASE_NAME}." ) @@ -285,6 +298,10 @@ def print_devserver_summary(state: DevserverState, *, app: str, no_reload: bool) print(f"Runtime root: {state.runtime_root}") if state.database_url: print(f"Database: {state.database_url}") + if state.database_url.startswith("postgresql"): + pgtools_url = os.getenv("GOVOPLAN_DATABASE_URL_PGTOOLS") + if pgtools_url: + print(f"PostgreSQL tools URL: {pgtools_url}") if state.bootstrap_db_path is not None: bootstrap_state = "enabled" if getattr(state.config.settings, "dev_bootstrap_enabled", False) else "disabled by DEV_BOOTSTRAP_ENABLED" print(f"Dev bootstrap for missing SQLite DB: {bootstrap_state} ({state.bootstrap_db_path})") diff --git a/src/govoplan_core/i18n.py b/src/govoplan_core/i18n.py new file mode 100644 index 0000000..5f45a16 --- /dev/null +++ b/src/govoplan_core/i18n.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import re +from typing import Any, Iterable + +I18N_SETTINGS_KEY = "i18n" + +DEFAULT_LANGUAGE_PACKAGES: tuple[dict[str, str], ...] = ( + {"code": "en", "label": "English", "native_label": "English"}, + {"code": "de", "label": "German", "native_label": "Deutsch"}, +) + + +def normalize_language_code(value: object) -> str: + raw = str(value or "").strip().lower().replace("_", "-") + parts = [part for part in re.split(r"[^a-z0-9]+", raw) if part] + return "-".join(parts) + + +def language_code_candidates(value: object) -> tuple[str, ...]: + code = normalize_language_code(value) + if not code: + return () + primary = code.split("-", 1)[0] + return tuple(dict.fromkeys((code, primary))) + + +def resolve_language_code(value: object, allowed_codes: Iterable[str]) -> str: + allowed = {normalize_language_code(code) for code in allowed_codes if normalize_language_code(code)} + for candidate in language_code_candidates(value): + if candidate in allowed: + return candidate + return "" + + +def i18n_settings(settings: dict[str, Any] | None) -> dict[str, Any]: + raw = (settings or {}).get(I18N_SETTINGS_KEY) + return dict(raw) if isinstance(raw, dict) else {} + + +def normalize_language_packages(raw_packages: object = None) -> list[dict[str, str]]: + packages: dict[str, dict[str, str]] = {} + for item in DEFAULT_LANGUAGE_PACKAGES: + packages[item["code"]] = dict(item) + + if isinstance(raw_packages, (list, tuple)): + for raw in raw_packages: + if not isinstance(raw, dict): + continue + code = normalize_language_code(raw.get("code")) + if not code: + continue + label = str(raw.get("label") or code.upper()).strip() + native_label = str(raw.get("native_label") or raw.get("nativeLabel") or label).strip() + packages[code] = {"code": code, "label": label, "native_label": native_label} + + return list(packages.values()) + + +def system_language_packages(settings: dict[str, Any] | None) -> list[dict[str, str]]: + return normalize_language_packages(i18n_settings(settings).get("available_languages")) + + +def normalize_enabled_language_codes( + codes: object, + available_languages: Iterable[dict[str, Any]], + *, + default_locale: object = None, + fallback_codes: Iterable[str] = ("en", "de"), +) -> list[str]: + available_codes = [normalize_language_code(item.get("code")) for item in available_languages if isinstance(item, dict)] + available = {code for code in available_codes if code} + requested: list[str] = [] + if isinstance(codes, (list, tuple, set)): + requested = [normalize_language_code(item) for item in codes] + if not requested: + requested = [normalize_language_code(item) for item in fallback_codes] + + enabled: list[str] = [] + for item in requested: + code = resolve_language_code(item, available) + if code and code not in enabled: + enabled.append(code) + + default_code = resolve_language_code(default_locale, available) + if default_code and default_code not in enabled: + enabled.insert(0, default_code) + + if enabled: + return enabled + if "en" in available: + return ["en"] + return available_codes[:1] + + +def system_enabled_language_codes(settings: dict[str, Any] | None, *, default_locale: object = None) -> list[str]: + i18n = i18n_settings(settings) + raw_enabled = i18n.get("enabled_language_codes", i18n.get("enabled_languages")) + return normalize_enabled_language_codes(raw_enabled, system_language_packages(settings), default_locale=default_locale) + + +def tenant_enabled_language_codes( + tenant_settings: dict[str, Any] | None, + system_enabled_codes: Iterable[str], + *, + default_locale: object = None, +) -> list[str]: + enabled = [normalize_language_code(code) for code in system_enabled_codes if normalize_language_code(code)] + available = [{"code": code} for code in enabled] + raw_enabled = i18n_settings(tenant_settings).get("enabled_language_codes") + return normalize_enabled_language_codes(raw_enabled, available, default_locale=default_locale, fallback_codes=enabled) + + +def user_enabled_language_codes(user_settings: dict[str, Any] | None, tenant_enabled_codes: Iterable[str]) -> list[str]: + enabled = [normalize_language_code(code) for code in tenant_enabled_codes if normalize_language_code(code)] + available = [{"code": code} for code in enabled] + raw_enabled = i18n_settings(user_settings).get("enabled_language_codes") + return normalize_enabled_language_codes(raw_enabled, available, fallback_codes=enabled) + + +def preferred_language_code(user_settings: dict[str, Any] | None, allowed_codes: Iterable[str], *, default_locale: object = None) -> str: + enabled = [normalize_language_code(code) for code in allowed_codes if normalize_language_code(code)] + preferred = resolve_language_code(i18n_settings(user_settings).get("preferred_language"), enabled) + if preferred: + return preferred + default = resolve_language_code(default_locale, enabled) + if default: + return default + return enabled[0] if enabled else "en" + + +def update_i18n_settings(settings: dict[str, Any] | None, **values: Any) -> dict[str, Any]: + updated = dict(settings or {}) + current = i18n_settings(updated) + current.update(values) + updated[I18N_SETTINGS_KEY] = current + return updated + + +def system_i18n_payload(settings_item: Any | None) -> dict[str, object]: + settings = getattr(settings_item, "settings", None) + packages = system_language_packages(settings) + available_codes = [item["code"] for item in packages] + default_language = resolve_language_code(getattr(settings_item, "default_locale", None), available_codes) or "en" + enabled = system_enabled_language_codes(settings, default_locale=default_language) + if default_language not in enabled: + default_language = enabled[0] if enabled else "en" + return { + "available_languages": packages, + "enabled_languages": enabled, + "default_language": default_language, + } diff --git a/src/govoplan_core/privacy/retention.py b/src/govoplan_core/privacy/retention.py index d906ed0..a609142 100644 --- a/src/govoplan_core/privacy/retention.py +++ b/src/govoplan_core/privacy/retention.py @@ -17,6 +17,7 @@ from govoplan_core.core.campaigns import ( CampaignPolicyContextProvider, CampaignRetentionProvider, ) +from govoplan_core.core.policy import policy_source_step from govoplan_core.core.runtime import get_registry from govoplan_access.backend.db.models import Group, User from govoplan_audit.backend.db.models import AuditLog @@ -339,13 +340,13 @@ def _retention_policy_source_fields(patch: dict[str, Any], *, baseline: bool = F def _retention_policy_source_step(scope_type: str, label: str, scope_id: str | None, patch: dict[str, Any], *, baseline: bool = False) -> dict[str, Any]: source_policy = PrivacyRetentionPolicy.model_validate(patch).model_dump(mode="json") if baseline else dict(patch) - return { - "scope_type": scope_type, - "scope_id": scope_id, - "label": label, - "applied_fields": _retention_policy_source_fields(patch, baseline=baseline), - "policy": source_policy, - } + return policy_source_step( + scope_type, + label, + scope_id, + applied_fields=_retention_policy_source_fields(patch, baseline=baseline), + policy=source_policy, + ) def effective_privacy_policy_sources( diff --git a/src/govoplan_core/security/secrets.py b/src/govoplan_core/security/secrets.py index 8d08138..a0230e7 100644 --- a/src/govoplan_core/security/secrets.py +++ b/src/govoplan_core/security/secrets.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 import hashlib from functools import lru_cache +from typing import Protocol, runtime_checkable from cryptography.fernet import Fernet, InvalidToken @@ -17,6 +18,21 @@ class SecretDecryptionError(RuntimeError): pass +CAPABILITY_SECURITY_SECRET_PROVIDER = "security.secretProvider" + + +@runtime_checkable +class SecretProvider(Protocol): + def store_secret(self, *, scope: str, name: str, value: str) -> str: + ... + + def read_secret(self, secret_ref: str) -> str | None: + ... + + def delete_secret(self, secret_ref: str) -> None: + ... + + def _normalize_fernet_key(value: str) -> bytes: candidate = value.strip().encode("utf-8") try: diff --git a/src/govoplan_core/server/app.py b/src/govoplan_core/server/app.py index 708714b..d8c9939 100644 --- a/src/govoplan_core/server/app.py +++ b/src/govoplan_core/server/app.py @@ -9,6 +9,7 @@ from govoplan_core.server.config import GovoplanServerConfig, load_server_config from govoplan_core.server.fastapi import create_govoplan_app from govoplan_core.server.platform import create_platform_router from govoplan_core.server.registry import available_module_manifests, build_platform_registry +from govoplan_core.server.route_validation import validate_no_route_collisions def create_app(config: GovoplanServerConfig | str | None = None): @@ -19,7 +20,8 @@ def create_app(config: GovoplanServerConfig | str | None = None): database_url = getattr(server_config.settings, "database_url", None) if server_config.settings is not None else None if database_url: - configure_database(str(database_url)) + dispose_previous = getattr(server_config.settings, "app_env", None) == "test" + configure_database(str(database_url), dispose_previous=dispose_previous) raw_enabled_modules = load_startup_enabled_modules(server_config.enabled_modules) candidate_modules = startup_candidate_module_ids(server_config.enabled_modules, raw_enabled_modules) @@ -55,6 +57,8 @@ def create_app(config: GovoplanServerConfig | str | None = None): if contribution.should_include(server_config.settings, registry): api_router.include_router(contribution.router) + validate_no_route_collisions(api_router, owner="server startup routes") + app = create_govoplan_app( title=server_config.title, version=server_config.version, diff --git a/src/govoplan_core/server/conditional_requests.py b/src/govoplan_core/server/conditional_requests.py new file mode 100644 index 0000000..06a78b7 --- /dev/null +++ b/src/govoplan_core/server/conditional_requests.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Awaitable, Callable + +from fastapi import Request +from starlette.responses import Response + +JSON_CACHE_CONTROL = "private, no-cache" +JSON_ETAG_VARY_HEADERS = ("Authorization", "Cookie", "X-API-Key", "Accept-Language") + + +async def conditional_json_get_middleware( + request: Request, + call_next: Callable[[Request], Awaitable[Response]], +) -> Response: + """Attach ETags to JSON GET responses and return 304 when unchanged. + + The middleware deliberately works after route handling. That keeps the + contract platform-wide without requiring every module router to learn about + conditional requests, while still limiting buffering to successful JSON GET + responses. + """ + + response = await call_next(request) + if not _eligible_for_conditional_json_get(request, response): + return response + + body = b"".join([chunk async for chunk in response.body_iterator]) + etag = response.headers.get("etag") or json_response_etag(body) + headers = dict(response.headers) + headers["etag"] = etag + headers["cache-control"] = _conditional_cache_control(headers.get("cache-control")) + headers["vary"] = _merge_vary(headers.get("vary"), JSON_ETAG_VARY_HEADERS) + headers.pop("content-length", None) + + if if_none_match_matches(request.headers.get("if-none-match"), etag): + return Response(status_code=304, headers=_not_modified_headers(headers), background=response.background) + + return Response(content=body, status_code=response.status_code, headers=headers, background=response.background) + + +def json_response_etag(body: bytes) -> str: + digest = hashlib.sha256(body).hexdigest() + return f'W/"sha256-{digest}"' + + +def if_none_match_matches(header_value: str | None, etag: str) -> bool: + if not header_value: + return False + normalized_etag = _normalize_etag_for_weak_compare(etag) + for raw_candidate in header_value.split(","): + candidate = raw_candidate.strip() + if candidate == "*": + return True + if _normalize_etag_for_weak_compare(candidate) == normalized_etag: + return True + return False + + +def _eligible_for_conditional_json_get(request: Request, response: Response) -> bool: + if request.method.upper() != "GET": + return False + if response.status_code != 200: + return False + if "set-cookie" in response.headers: + return False + if "content-encoding" in response.headers: + return False + if "content-disposition" in response.headers: + return False + if "no-store" in request.headers.get("cache-control", "").lower(): + return False + if "no-store" in response.headers.get("cache-control", "").lower(): + return False + return _is_json_content_type(response.headers.get("content-type")) + + +def _is_json_content_type(value: str | None) -> bool: + media_type = (value or "").split(";", 1)[0].strip().lower() + return media_type == "application/json" or media_type.endswith("+json") + + +def _conditional_cache_control(current: str | None) -> str: + if not current: + return JSON_CACHE_CONTROL + directives = [item.strip() for item in current.split(",") if item.strip()] + normalized = {item.split("=", 1)[0].strip().lower() for item in directives} + if "private" not in normalized and "public" not in normalized: + directives.append("private") + if "no-cache" not in normalized: + directives.append("no-cache") + return ", ".join(directives) + + +def _merge_vary(current: str | None, additions: tuple[str, ...]) -> str: + tokens: list[str] = [] + seen: set[str] = set() + for value in [current or "", *additions]: + for token in value.split(","): + stripped = token.strip() + if not stripped: + continue + normalized = stripped.lower() + if normalized in seen: + continue + seen.add(normalized) + tokens.append(stripped) + return ", ".join(tokens) + + +def _not_modified_headers(headers: dict[str, str]) -> dict[str, str]: + allowed = {"cache-control", "content-location", "date", "etag", "expires", "vary", "x-correlation-id"} + return {key: value for key, value in headers.items() if key.lower() in allowed} + + +def _normalize_etag_for_weak_compare(value: str) -> str: + normalized = value.strip() + if normalized.startswith("W/"): + normalized = normalized[2:].strip() + if len(normalized) >= 2 and normalized[0] == '"' and normalized[-1] == '"': + normalized = normalized[1:-1] + return normalized diff --git a/src/govoplan_core/server/default_config.py b/src/govoplan_core/server/default_config.py index c1ae028..a6dc02a 100644 --- a/src/govoplan_core/server/default_config.py +++ b/src/govoplan_core/server/default_config.py @@ -3,8 +3,9 @@ from __future__ import annotations from contextlib import asynccontextmanager from fastapi import Depends, FastAPI +from sqlalchemy.engine import make_url -from govoplan_access.backend.auth.dependencies import ApiPrincipal, require_scope +from govoplan_access.auth import ApiPrincipal, require_scope from govoplan_core.core.registry import PlatformRegistry from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables from govoplan_core.db.session import get_database @@ -12,6 +13,13 @@ from govoplan_core.server.config import GovoplanServerConfig from govoplan_core.settings import Settings, settings +def _dev_bootstrap_needs_create_all(database_url: str) -> bool: + try: + return make_url(database_url).get_backend_name() == "sqlite" + except Exception: + return database_url.startswith("sqlite:") + + @asynccontextmanager async def lifespan(app: FastAPI): if settings.app_env.lower() == "dev" and settings.dev_auto_migrate_enabled: @@ -19,7 +27,8 @@ async def lifespan(app: FastAPI): migrate_database(database_url=settings.database_url) if settings.app_env.lower() == "dev" and settings.dev_bootstrap_enabled: - create_all_tables() + if _dev_bootstrap_needs_create_all(settings.database_url): + create_all_tables() with get_database().SessionLocal() as session: bootstrap_dev_data( session, diff --git a/src/govoplan_core/server/fastapi.py b/src/govoplan_core/server/fastapi.py index 19b98e0..ca2853d 100644 --- a/src/govoplan_core/server/fastapi.py +++ b/src/govoplan_core/server/fastapi.py @@ -4,10 +4,12 @@ from collections.abc import AsyncIterator, Callable, Iterable from contextlib import AbstractAsyncContextManager from typing import Any -from fastapi import APIRouter, FastAPI +from fastapi import APIRouter, FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from govoplan_core.core.events import event_context, new_event_id, normalize_trace_id from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.server.conditional_requests import conditional_json_get_middleware LifespanFactory = Callable[[FastAPI], AbstractAsyncContextManager[None] | AsyncIterator[None]] @@ -25,6 +27,20 @@ def create_govoplan_app( app = FastAPI(title=title, version=version, lifespan=lifespan) app.state.govoplan_registry = registry + @app.middleware("http") + async def request_correlation_context(request: Request, call_next): + correlation_id = ( + normalize_trace_id(request.headers.get("x-correlation-id")) + or normalize_trace_id(request.headers.get("x-request-id")) + or new_event_id() + ) + with event_context(correlation_id=correlation_id): + response = await call_next(request) + response.headers["X-Correlation-ID"] = correlation_id + return response + + app.middleware("http")(conditional_json_get_middleware) + origins = [item.strip() for item in cors_origins if item.strip()] if origins: app.add_middleware( diff --git a/src/govoplan_core/server/platform.py b/src/govoplan_core/server/platform.py index b6e4284..45ac7b1 100644 --- a/src/govoplan_core/server/platform.py +++ b/src/govoplan_core/server/platform.py @@ -3,10 +3,13 @@ from __future__ import annotations from fastapi import APIRouter, HTTPException, Request from sqlalchemy.exc import SQLAlchemyError +from govoplan_core.admin.models import SystemSettings +from govoplan_core.admin.settings import SYSTEM_SETTINGS_ID from govoplan_core.core.maintenance import saved_maintenance_mode from govoplan_core.core.modules import FrontendModule, FrontendRoute, NavItem from govoplan_core.core.registry import PlatformRegistry from govoplan_core.db.session import get_database +from govoplan_core.i18n import system_i18n_payload def _registry(request: Request) -> PlatformRegistry: @@ -72,10 +75,13 @@ def create_platform_router(settings: object | None = None) -> APIRouter: try: with get_database().session() as session: maintenance_mode = saved_maintenance_mode(session) + settings_item = session.get(SystemSettings, SYSTEM_SETTINGS_ID) except (RuntimeError, SQLAlchemyError): maintenance_mode = None + settings_item = None return { "maintenance_mode": maintenance_mode.as_dict() if maintenance_mode is not None else {"enabled": False, "message": None}, + "i18n": system_i18n_payload(settings_item), } @router.get("/modules") diff --git a/src/govoplan_core/server/route_validation.py b/src/govoplan_core/server/route_validation.py new file mode 100644 index 0000000..7dfd18e --- /dev/null +++ b/src/govoplan_core/server/route_validation.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass + + +class RouteCollisionError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class RouteSignature: + method: str + path: str + + +@dataclass(frozen=True, slots=True) +class RouteRecord: + signature: RouteSignature + owner: str + + +def validate_no_route_collisions(routes_or_app: object, *, owner: str = "router") -> None: + seen: dict[RouteSignature, RouteRecord] = {} + for record in iter_route_records(routes_or_app, owner=owner): + previous = seen.get(record.signature) + if previous is not None: + raise RouteCollisionError(_collision_message(record.signature, previous.owner, record.owner)) + seen[record.signature] = record + + +def validate_router_can_mount(existing_routes_or_app: object, candidate_router: object, *, prefix: str = "", owner: str) -> None: + validate_no_route_collisions(candidate_router, owner=owner) + existing = {record.signature: record for record in iter_route_records(existing_routes_or_app, owner="existing application")} + for record in iter_route_records(candidate_router, prefix=prefix, owner=owner): + previous = existing.get(record.signature) + if previous is not None: + raise RouteCollisionError(_collision_message(record.signature, previous.owner, record.owner)) + + +def iter_route_records(routes_or_app: object, *, prefix: str = "", owner: str = "router") -> Iterable[RouteRecord]: + routes = getattr(routes_or_app, "routes", routes_or_app) + for route in routes or (): + path = _join_route_path(prefix, str(getattr(route, "path", "") or "")) + include_context = getattr(route, "include_context", None) + nested_prefix = _join_route_path(prefix, str(getattr(include_context, "prefix", "") or "")) + nested_router = getattr(route, "original_router", None) + if nested_router is not None: + yield from iter_route_records(nested_router, prefix=nested_prefix, owner=owner) + + methods = getattr(route, "methods", None) + if methods: + for method in sorted(str(item).upper() for item in methods): + yield RouteRecord(RouteSignature(method=method, path=path), owner) + + nested_routes = getattr(route, "routes", None) + if nested_routes is not None and nested_routes is not routes: + yield from iter_route_records(nested_routes, prefix=path, owner=owner) + + +def _join_route_path(prefix: str, path: str) -> str: + if not prefix: + return path or "/" + if not path: + return prefix or "/" + return f"{prefix.rstrip('/')}/{path.lstrip('/')}" or "/" + + +def _collision_message(signature: RouteSignature, previous_owner: str, next_owner: str) -> str: + return ( + f"Route collision: {signature.method} {signature.path} is registered by " + f"{previous_owner} and {next_owner}" + ) diff --git a/src/govoplan_core/settings.py b/src/govoplan_core/settings.py index e0cb637..45b26e5 100644 --- a/src/govoplan_core/settings.py +++ b/src/govoplan_core/settings.py @@ -8,7 +8,7 @@ class Settings(BaseSettings): app_env: str = Field(default="dev", alias="APP_ENV") database_url: str = Field( - default="sqlite:///./runtime/multimailer-dev.db", + default="postgresql+psycopg://govoplan_dev@127.0.0.1:5432/govoplan_dev", alias="DATABASE_URL", ) access_database_url: str | None = Field(default=None, alias="ACCESS_DATABASE_URL") @@ -17,7 +17,7 @@ class Settings(BaseSettings): tenant_data_mode: str = Field(default="shared", alias="TENANT_DATA_MODE") tenant_db_url_template: str | None = Field(default=None, alias="TENANT_DB_URL_TEMPLATE") tenant_schema_template: str | None = Field(default=None, alias="TENANT_SCHEMA_TEMPLATE") - enabled_modules: str = Field(default="tenancy,access,admin,policy,audit,campaigns,files,mail,calendar", alias="ENABLED_MODULES") + enabled_modules: str = Field(default="tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,docs,ops", alias="ENABLED_MODULES") redis_url: str = Field(default="redis://redis:6379/0", alias="REDIS_URL") celery_enabled: bool = Field(default=False, alias="CELERY_ENABLED") diff --git a/tests/db_isolation.py b/tests/db_isolation.py new file mode 100644 index 0000000..7f17946 --- /dev/null +++ b/tests/db_isolation.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from govoplan_core.db.session import DatabaseHandle, configure_database, get_database, reset_database, set_database + + +@contextmanager +def temporary_database(database_url: str) -> Iterator[DatabaseHandle]: + try: + previous_database = get_database() + except RuntimeError: + previous_database = None + + database = configure_database(database_url) + should_dispose = database is not previous_database + try: + yield database + finally: + if should_dispose: + database.engine.dispose() + if previous_database is None: + reset_database() + else: + set_database(previous_database) diff --git a/tests/test_access_contracts.py b/tests/test_access_contracts.py index ac0a04b..6320538 100644 --- a/tests/test_access_contracts.py +++ b/tests/test_access_contracts.py @@ -6,10 +6,10 @@ import unittest from pathlib import Path from types import SimpleNamespace -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, FastAPI from fastapi.testclient import TestClient -from govoplan_access.backend.auth.dependencies import ApiPrincipal, get_api_principal +from govoplan_access.auth import ApiPrincipal, get_api_principal from govoplan_core.core.access import ( ACCESS_CAPABILITY_NAMES, ACCESS_MODULE_ID, @@ -19,32 +19,41 @@ from govoplan_core.core.access import ( CAPABILITY_ACCESS_PERMISSION_EVALUATOR, CAPABILITY_ACCESS_PRINCIPAL_RESOLVER, CAPABILITY_ACCESS_RESOURCE_ACCESS, + CAPABILITY_ACCESS_EXPLANATION, + CAPABILITY_ACCESS_SEMANTIC_DIRECTORY, CAPABILITY_ACCESS_TENANT_PROVISIONER, CAPABILITY_AUDIT_SINK, - CAPABILITY_SECURITY_SECRET_PROVIDER, CAPABILITY_TENANCY_TENANT_RESOLVER, AccessDecision, AccessAdministration, AccessDirectory, + AccessDecisionProvenance, + AccessExplanationService, AccessGovernanceMaterializer, + AccessSemanticDirectory, AccessSubjectRef, AccountRef, AuditEvent, AuditSink, + FunctionAssignmentRef, + FunctionDelegationRef, + FunctionRef, GovernanceTemplateMaterialization, GroupRef, + IdentityRef, + OrganizationUnitRef, PermissionEvaluator, PrincipalRef, PrincipalResolver, ResourceAccessService, RoleRef, - SecretProvider, TenantRef, TenantAccessProvisioner, TenantOwnerCandidateRef, TenantResolver, UserRef, ) +from govoplan_core.security.secrets import CAPABILITY_SECURITY_SECRET_PROVIDER, SecretProvider from govoplan_core.core.campaigns import ( CAPABILITY_CAMPAIGNS_ACCESS, CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, @@ -62,11 +71,23 @@ from govoplan_core.core.campaigns import ( from govoplan_core.core.modules import ModuleContext, ModuleManifest from govoplan_core.core.registry import PlatformRegistry from govoplan_core.db.base import Base -from govoplan_core.db.session import configure_database, get_database from govoplan_core.server.app import create_app from govoplan_core.server.config import GovoplanServerConfig -from govoplan_access.backend.db.models import Account, Group, User, UserGroupMembership +from govoplan_access.backend.db.models import ( + Account, + Function, + FunctionAssignment, + FunctionRoleAssignment, + Group, + Identity, + IdentityAccountLink, + OrganizationUnit, + Role, + User, + UserGroupMembership, +) from govoplan_tenancy.backend.db.models import Tenant +from tests.db_isolation import temporary_database class _FakeAccessDirectory: @@ -108,6 +129,64 @@ class _FakeAccessDirectory: return subject.label +class _FakeAccessSemanticDirectory(_FakeAccessDirectory): + identity = IdentityRef( + id="identity-1", + display_name="Demo Person", + primary_account_id=_FakeAccessDirectory.account.id, + account_ids=(_FakeAccessDirectory.account.id, "account-2"), + ) + organization_unit = OrganizationUnitRef(id="ou-1", tenant_id="tenant-1", name="Registry Office") + function = FunctionRef( + id="function-1", + tenant_id="tenant-1", + organization_unit_id=organization_unit.id, + slug="registry-clerk", + name="Registry Clerk", + role_ids=("role-1",), + delegable=True, + act_in_place_allowed=True, + ) + assignment = FunctionAssignmentRef( + id="assignment-1", + tenant_id="tenant-1", + account_id=_FakeAccessDirectory.account.id, + identity_id=identity.id, + function_id=function.id, + organization_unit_id=organization_unit.id, + applies_to_subunits=True, + ) + + def get_identity(self, identity_id: str) -> IdentityRef | None: + return self.identity if identity_id == self.identity.id else None + + def accounts_for_identity(self, identity_id: str): + return (self.account,) if identity_id == self.identity.id else () + + def get_organization_unit(self, organization_unit_id: str) -> OrganizationUnitRef | None: + return self.organization_unit if organization_unit_id == self.organization_unit.id else None + + def organization_units_for_tenant(self, tenant_id: str): + return (self.organization_unit,) if tenant_id == self.organization_unit.tenant_id else () + + def get_function(self, function_id: str) -> FunctionRef | None: + return self.function if function_id == self.function.id else None + + def functions_for_organization_unit(self, organization_unit_id: str, *, include_subunits: bool = False): + del include_subunits + return (self.function,) if organization_unit_id == self.organization_unit.id else () + + def get_function_assignment(self, assignment_id: str) -> FunctionAssignmentRef | None: + return self.assignment if assignment_id == self.assignment.id else None + + def function_assignments_for_account(self, account_id: str, *, tenant_id: str | None = None): + if account_id != self.account.id: + return () + if tenant_id is not None and tenant_id != self.assignment.tenant_id: + return () + return (self.assignment,) + + class _FakePrincipalResolver: def resolve_request(self, request: object, *, session: object | None = None) -> PrincipalRef: del request, session @@ -148,6 +227,30 @@ class _FakeResourceAccessService: return AccessDecision(allowed=True) +class _FakeAccessExplanationService: + provenance = ( + AccessDecisionProvenance(kind="identity", id="identity-1", label="Demo Person"), + AccessDecisionProvenance(kind="function", id="assignment-1", label="Registry Clerk", tenant_id="tenant-1"), + AccessDecisionProvenance(kind="role", id="role-1", label="Clerk", tenant_id="tenant-1"), + AccessDecisionProvenance(kind="right", id="files:file:read", label="Read files"), + ) + + def explain_scope_provenance(self, principal: PrincipalRef, required_scope: str): + del principal, required_scope + return self.provenance + + def explain_resource_provenance( + self, + principal: PrincipalRef, + *, + resource_type: str, + resource_id: str, + action: str, + ): + del principal, resource_type, resource_id, action + return self.provenance + + class _FakeTenantAccessProvisioner: def ensure_default_roles(self, session: object, tenant: object | None = None): del session, tenant @@ -311,6 +414,8 @@ class AccessContractTests(unittest.TestCase): self.assertEqual("access.directory", CAPABILITY_ACCESS_DIRECTORY) self.assertEqual("access.permissionEvaluator", CAPABILITY_ACCESS_PERMISSION_EVALUATOR) self.assertEqual("access.resourceAccess", CAPABILITY_ACCESS_RESOURCE_ACCESS) + self.assertEqual("access.semanticDirectory", CAPABILITY_ACCESS_SEMANTIC_DIRECTORY) + self.assertEqual("access.explanation", CAPABILITY_ACCESS_EXPLANATION) self.assertEqual("access.tenantProvisioner", CAPABILITY_ACCESS_TENANT_PROVISIONER) self.assertEqual("access.administration", CAPABILITY_ACCESS_ADMINISTRATION) self.assertEqual("access.governanceMaterializer", CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER) @@ -329,8 +434,13 @@ class AccessContractTests(unittest.TestCase): account_id="account-1", membership_id="user-1", tenant_id="tenant-1", + identity_id="identity-1", scopes=frozenset({"campaigns:campaign:read"}), group_ids=frozenset({"group-1"}), + role_ids=frozenset({"role-1"}), + function_assignment_ids=frozenset({"assignment-1"}), + delegation_ids=frozenset({"delegation-1"}), + acting_for_account_id="account-2", email="demo@example.test", ) role = RoleRef(id="role-1", name="Reviewer", level="tenant", tenant_id="tenant-1") @@ -338,15 +448,80 @@ class AccessContractTests(unittest.TestCase): self.assertEqual("account-1", principal.account_id) self.assertEqual(frozenset({"campaigns:campaign:read"}), principal.scopes) self.assertEqual("Reviewer", role.name) + payload = principal.to_dict() + self.assertEqual(["campaigns:campaign:read"], payload["scopes"]) + self.assertEqual(["group-1"], payload["group_ids"]) + self.assertEqual(["role-1"], payload["role_ids"]) + self.assertEqual(["assignment-1"], payload["function_assignment_ids"]) + self.assertEqual(["delegation-1"], payload["delegation_ids"]) + self.assertEqual("identity-1", payload["identity_id"]) + self.assertEqual("account-2", payload["acting_for_account_id"]) + self.assertEqual("session", payload["auth_method"]) + self.assertIsNone(payload["api_key_id"]) + self.assertIsNone(payload["service_account_id"]) + self.assertEqual(principal, PrincipalRef.from_mapping(payload)) with self.assertRaises(AttributeError): principal.account_id = "changed" # type: ignore[misc] + def test_identity_function_and_delegation_refs_model_organization_scope(self) -> None: + identity = IdentityRef( + id="identity-1", + display_name="Demo Person", + primary_account_id="account-1", + account_ids=("account-1", "account-2"), + ) + organization_unit = OrganizationUnitRef(id="ou-1", tenant_id="tenant-1", name="Registry Office") + function = FunctionRef( + id="function-1", + tenant_id="tenant-1", + organization_unit_id=organization_unit.id, + slug="registry-clerk", + name="Registry Clerk", + role_ids=("role-1",), + delegable=True, + act_in_place_allowed=True, + ) + assignment = FunctionAssignmentRef( + id="assignment-1", + tenant_id="tenant-1", + account_id="account-1", + identity_id=identity.id, + function_id=function.id, + organization_unit_id=organization_unit.id, + applies_to_subunits=True, + ) + delegation = FunctionDelegationRef( + id="delegation-1", + tenant_id="tenant-1", + function_assignment_id=assignment.id, + delegator_account_id="account-1", + delegate_account_id="account-2", + mode="act_in_place", + ) + provenance = AccessDecisionProvenance( + kind="function", + id=assignment.id, + label=function.name, + tenant_id="tenant-1", + source="delegation", + details={"organization_unit_id": organization_unit.id, "applies_to_subunits": True}, + ) + + self.assertEqual(("account-1", "account-2"), identity.account_ids) + self.assertEqual("ou-1", function.organization_unit_id) + self.assertTrue(assignment.applies_to_subunits) + self.assertEqual("act_in_place", delegation.mode) + self.assertEqual("function", provenance.to_dict()["kind"]) + self.assertEqual({"organization_unit_id": "ou-1", "applies_to_subunits": True}, provenance.to_dict()["details"]) + def test_protocol_shapes_match_reference_implementations(self) -> None: self.assertIsInstance(_FakePrincipalResolver(), PrincipalResolver) self.assertIsInstance(_FakeTenantResolver(), TenantResolver) self.assertIsInstance(_FakeAccessDirectory(), AccessDirectory) + self.assertIsInstance(_FakeAccessSemanticDirectory(), AccessSemanticDirectory) self.assertIsInstance(_FakePermissionEvaluator(), PermissionEvaluator) self.assertIsInstance(_FakeResourceAccessService(), ResourceAccessService) + self.assertIsInstance(_FakeAccessExplanationService(), AccessExplanationService) self.assertIsInstance(_FakeTenantAccessProvisioner(), TenantAccessProvisioner) self.assertIsInstance(_FakeAccessAdministration(), AccessAdministration) self.assertIsInstance(_FakeAccessGovernanceMaterializer(), AccessGovernanceMaterializer) @@ -389,6 +564,8 @@ class AccessContractTests(unittest.TestCase): def test_access_capabilities_register_and_resolve_through_platform_registry(self) -> None: directory = _FakeAccessDirectory() + semantic_directory = _FakeAccessSemanticDirectory() + explanation = _FakeAccessExplanationService() resolver = _FakePrincipalResolver() evaluator = _FakePermissionEvaluator() tenant_resolver = _FakeTenantResolver() @@ -400,6 +577,8 @@ class AccessContractTests(unittest.TestCase): version="test", capability_factories={ CAPABILITY_ACCESS_DIRECTORY: lambda context: directory, + CAPABILITY_ACCESS_SEMANTIC_DIRECTORY: lambda context: semantic_directory, + CAPABILITY_ACCESS_EXPLANATION: lambda context: explanation, CAPABILITY_ACCESS_PRINCIPAL_RESOLVER: lambda context: resolver, CAPABILITY_ACCESS_PERMISSION_EVALUATOR: lambda context: evaluator, CAPABILITY_ACCESS_ADMINISTRATION: lambda context: administration, @@ -413,6 +592,8 @@ class AccessContractTests(unittest.TestCase): registry.configure_capability_context(ModuleContext(registry=registry, settings=object())) self.assertIs(directory, registry.require_capability(CAPABILITY_ACCESS_DIRECTORY)) + self.assertIs(semantic_directory, registry.require_capability(CAPABILITY_ACCESS_SEMANTIC_DIRECTORY)) + self.assertIs(explanation, registry.require_capability(CAPABILITY_ACCESS_EXPLANATION)) self.assertIs(resolver, registry.require_capability(CAPABILITY_ACCESS_PRINCIPAL_RESOLVER)) self.assertIs(evaluator, registry.require_capability(CAPABILITY_ACCESS_PERMISSION_EVALUATOR)) self.assertIs(administration, registry.require_capability(CAPABILITY_ACCESS_ADMINISTRATION)) @@ -422,50 +603,240 @@ class AccessContractTests(unittest.TestCase): def test_sql_directory_and_tenant_resolver_return_access_dtos(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-access-directory-")) try: - configure_database(f"sqlite:///{root / 'directory.db'}") - database = get_database() - Base.metadata.create_all(bind=database.engine) - with database.session() as session: - tenant = Tenant(id="tenant-directory", slug="directory", name="Directory Tenant", settings={}) - account = Account( - id="account-directory", - email="directory@example.test", - normalized_email="directory@example.test", - display_name="Directory User", + with temporary_database(f"sqlite:///{root / 'directory.db'}") as database: + Base.metadata.create_all(bind=database.engine) + with database.session() as session: + tenant = Tenant(id="tenant-directory", slug="directory", name="Directory Tenant", settings={}) + account = Account( + id="account-directory", + email="directory@example.test", + normalized_email="directory@example.test", + display_name="Directory User", + ) + user = User( + id="user-directory", + tenant_id=tenant.id, + account_id=account.id, + email=account.email, + display_name=None, + settings={}, + mail_profile_policy={}, + ) + group = Group(id="group-directory", tenant_id=tenant.id, slug="review", name="Review", description=None) + membership = UserGroupMembership(tenant_id=tenant.id, user_id=user.id, group_id=group.id) + identity = Identity(id="identity-directory", display_name="Directory Person", source="local") + identity_link = IdentityAccountLink( + identity_id=identity.id, + account_id=account.id, + is_primary=True, + source="local", + ) + organization_unit = OrganizationUnit( + id="ou-directory", + tenant_id=tenant.id, + slug="registry", + name="Registry", + ) + role = Role( + id="role-directory", + tenant_id=tenant.id, + slug="registry-reader", + name="Registry Reader", + permissions=["files:read"], + ) + function = Function( + id="function-directory", + tenant_id=tenant.id, + organization_unit_id=organization_unit.id, + slug="registry-clerk", + name="Registry Clerk", + delegable=True, + ) + function_role = FunctionRoleAssignment(tenant_id=tenant.id, function_id=function.id, role_id=role.id) + function_assignment = FunctionAssignment( + id="assignment-directory", + tenant_id=tenant.id, + account_id=account.id, + identity_id=identity.id, + function_id=function.id, + organization_unit_id=organization_unit.id, + applies_to_subunits=True, + ) + session.add_all([ + tenant, + account, + user, + group, + membership, + identity, + identity_link, + organization_unit, + role, + function, + function_role, + function_assignment, + ]) + session.commit() + + from govoplan_access.backend.directory import SqlAccessDirectory + from govoplan_access.backend.security.sessions import collect_user_scopes + from govoplan_tenancy.backend.capabilities import SqlTenantResolver + + directory = SqlAccessDirectory() + tenant_resolver = SqlTenantResolver() + + self.assertEqual("directory@example.test", directory.get_account("account-directory").email) # type: ignore[union-attr] + self.assertEqual("Directory User", directory.get_user("user-directory").display_name) # type: ignore[union-attr] + self.assertEqual(["user-directory"], [user.id for user in directory.users_for_tenant("tenant-directory")]) + self.assertEqual(["group-directory"], [group.id for group in directory.groups_for_tenant("tenant-directory")]) + self.assertEqual(["group-directory"], [group.id for group in directory.groups_for_user("user-directory", tenant_id="tenant-directory")]) + self.assertEqual("Review", directory.display_label(AccessSubjectRef(kind="group", id="group-directory", tenant_id="tenant-directory"))) + self.assertEqual("Directory Person", directory.get_identity("identity-directory").display_name) # type: ignore[union-attr] + self.assertEqual(["account-directory"], [account.id for account in directory.accounts_for_identity("identity-directory")]) + self.assertEqual("Registry", directory.get_organization_unit("ou-directory").name) # type: ignore[union-attr] + self.assertEqual(["role-directory"], list(directory.get_function("function-directory").role_ids)) # type: ignore[union-attr] + self.assertEqual( + ["assignment-directory"], + [item.id for item in directory.function_assignments_for_account("account-directory", tenant_id="tenant-directory")], ) - user = User( - id="user-directory", - tenant_id=tenant.id, - account_id=account.id, - email=account.email, - display_name=None, - settings={}, - mail_profile_policy={}, + with database.session() as session: + user = session.get(User, "user-directory") + self.assertIn("files:read", collect_user_scopes(session, user, include_system=False)) # type: ignore[arg-type] + self.assertEqual("directory", tenant_resolver.get_tenant("tenant-directory").slug) # type: ignore[union-attr] + self.assertEqual( + "tenant-directory", + tenant_resolver.current_tenant( + PrincipalRef(account_id="account-directory", membership_id="user-directory", tenant_id="tenant-directory") + ).id, # type: ignore[union-attr] ) - group = Group(id="group-directory", tenant_id=tenant.id, slug="review", name="Review", description=None) - membership = UserGroupMembership(tenant_id=tenant.id, user_id=user.id, group_id=group.id) - session.add_all([tenant, account, user, group, membership]) - session.commit() + finally: + shutil.rmtree(root, ignore_errors=True) - from govoplan_access.backend.directory import SqlAccessDirectory - from govoplan_tenancy.backend.capabilities import SqlTenantResolver + def test_semantic_access_admin_crud_endpoints(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-access-semantic-api-")) + try: + with temporary_database(f"sqlite:///{root / 'semantic.db'}") as database: + Base.metadata.create_all(bind=database.engine) + tenant_id = "tenant-semantic" + account_id = "account-semantic" + delegate_account_id = "account-semantic-delegate" + user_id = "user-semantic" + role_id = "role-semantic" + with database.session() as session: + tenant = Tenant(id=tenant_id, slug="semantic", name="Semantic Tenant", settings={}) + account = Account( + id=account_id, + email="semantic@example.test", + normalized_email="semantic@example.test", + display_name="Semantic User", + ) + delegate_account = Account( + id=delegate_account_id, + email="delegate@example.test", + normalized_email="delegate@example.test", + display_name="Delegate User", + ) + user = User( + id=user_id, + tenant_id=tenant_id, + account_id=account_id, + email=account.email, + display_name=account.display_name, + settings={}, + mail_profile_policy={}, + ) + role = Role( + id=role_id, + tenant_id=tenant_id, + slug="semantic-reader", + name="Semantic Reader", + permissions=["files:read"], + ) + session.add_all([tenant, account, delegate_account, user, role]) + session.commit() - directory = SqlAccessDirectory() - tenant_resolver = SqlTenantResolver() + from govoplan_access.backend.api.v1.routes import router as admin_router - self.assertEqual("directory@example.test", directory.get_account("account-directory").email) # type: ignore[union-attr] - self.assertEqual("Directory User", directory.get_user("user-directory").display_name) # type: ignore[union-attr] - self.assertEqual(["user-directory"], [user.id for user in directory.users_for_tenant("tenant-directory")]) - self.assertEqual(["group-directory"], [group.id for group in directory.groups_for_tenant("tenant-directory")]) - self.assertEqual(["group-directory"], [group.id for group in directory.groups_for_user("user-directory", tenant_id="tenant-directory")]) - self.assertEqual("Review", directory.display_label(AccessSubjectRef(kind="group", id="group-directory", tenant_id="tenant-directory"))) - self.assertEqual("directory", tenant_resolver.get_tenant("tenant-directory").slug) # type: ignore[union-attr] - self.assertEqual( - "tenant-directory", - tenant_resolver.current_tenant( - PrincipalRef(account_id="account-directory", membership_id="user-directory", tenant_id="tenant-directory") - ).id, # type: ignore[union-attr] - ) + app = FastAPI() + app.include_router(admin_router) + + def principal_override() -> ApiPrincipal: + with database.session() as session: + account = session.get(Account, account_id) + user = session.get(User, user_id) + assert account is not None + assert user is not None + return ApiPrincipal( + principal=PrincipalRef( + account_id=account_id, + membership_id=user_id, + tenant_id=tenant_id, + scopes=frozenset({ + "admin:roles:read", + "admin:roles:write", + "admin:roles:assign", + "system:accounts:read", + "system:accounts:update", + }), + ), + account=account, + user=user, + ) + + app.dependency_overrides[get_api_principal] = principal_override + + with TestClient(app) as client: + identity_response = client.post( + "/admin/identities", + json={"display_name": "Semantic Person", "account_ids": [account_id], "primary_account_id": account_id}, + ) + self.assertEqual(201, identity_response.status_code, identity_response.text) + identity_id = identity_response.json()["id"] + + unit_response = client.post( + "/admin/organization-units", + json={"slug": "registry", "name": "Registry"}, + ) + self.assertEqual(201, unit_response.status_code, unit_response.text) + organization_unit_id = unit_response.json()["id"] + + function_response = client.post( + "/admin/functions", + json={ + "organization_unit_id": organization_unit_id, + "slug": "registry-clerk", + "name": "Registry Clerk", + "role_ids": [role_id], + "delegable": True, + }, + ) + self.assertEqual(201, function_response.status_code, function_response.text) + function_id = function_response.json()["id"] + self.assertEqual([role_id], function_response.json()["role_ids"]) + + assignment_response = client.post( + "/admin/function-assignments", + json={ + "account_id": account_id, + "identity_id": identity_id, + "function_id": function_id, + "applies_to_subunits": True, + }, + ) + self.assertEqual(201, assignment_response.status_code, assignment_response.text) + assignment_id = assignment_response.json()["id"] + self.assertTrue(assignment_response.json()["applies_to_subunits"]) + + delegation_response = client.post( + "/admin/function-delegations", + json={"function_assignment_id": assignment_id, "delegate_account_id": delegate_account_id}, + ) + self.assertEqual(201, delegation_response.status_code, delegation_response.text) + self.assertEqual(delegate_account_id, delegation_response.json()["delegate_account_id"]) + + list_response = client.get(f"/admin/function-assignments?account_id={account_id}") + self.assertEqual(200, list_response.status_code, list_response.text) + self.assertEqual([assignment_id], [item["id"] for item in list_response.json()["assignments"]]) finally: shutil.rmtree(root, ignore_errors=True) @@ -532,32 +903,32 @@ class AccessContractTests(unittest.TestCase): lifespan=None, app_configurators=(), ) - app = create_app(config) + with temporary_database(f"sqlite:///{root / 'test.db'}") as database: + app = create_app(config) - database = get_database() - Base.metadata.create_all(bind=database.engine) - with database.session() as session: - tenant = Tenant(id=tenant_id, slug="capability", name="Capability Tenant", settings={}) - account = Account( - id=account_id, - email="capability@example.test", - normalized_email="capability@example.test", - display_name="Capability User", - ) - user = User( - id=user_id, - tenant_id=tenant_id, - account_id=account_id, - email=account.email, - display_name=account.display_name, - settings={}, - mail_profile_policy={}, - ) - session.add_all([tenant, account, user]) - session.commit() + Base.metadata.create_all(bind=database.engine) + with database.session() as session: + tenant = Tenant(id=tenant_id, slug="capability", name="Capability Tenant", settings={}) + account = Account( + id=account_id, + email="capability@example.test", + normalized_email="capability@example.test", + display_name="Capability User", + ) + user = User( + id=user_id, + tenant_id=tenant_id, + account_id=account_id, + email=account.email, + display_name=account.display_name, + settings={}, + mail_profile_policy={}, + ) + session.add_all([tenant, account, user]) + session.commit() - with TestClient(app) as client: - response = client.get("/api/v1/capability-principal") + with TestClient(app) as client: + response = client.get("/api/v1/capability-principal") self.assertEqual(response.status_code, 200, response.text) self.assertEqual( diff --git a/tests/test_api_smoke.py b/tests/test_api_smoke.py index 2fe7816..d1aa9a8 100644 --- a/tests/test_api_smoke.py +++ b/tests/test_api_smoke.py @@ -5,12 +5,14 @@ import json import os import shutil import tempfile +import time import unittest import zipfile from datetime import datetime, timezone from email import policy from email.parser import BytesParser from pathlib import Path +from unittest.mock import patch import pyzipper @@ -28,7 +30,9 @@ from fastapi.testclient import TestClient from govoplan_core.db.base import Base from govoplan_core.db.bootstrap import bootstrap_dev_data -from govoplan_core.db.session import configure_database +from govoplan_core.db.session import configure_database, set_database +from govoplan_core.core.change_sequence import decode_sequence_watermark, prune_sequence_entries +from govoplan_core.core.pagination import encode_keyset_cursor, keyset_query_fingerprint _database = configure_database(os.environ["DATABASE_URL"]) engine = _database.engine @@ -50,6 +54,7 @@ class ApiSmokeTests(unittest.TestCase): shutil.rmtree(_TEST_ROOT, ignore_errors=True) def setUp(self) -> None: + set_database(_database) self.client.cookies.clear() Base.metadata.drop_all(bind=engine) Base.metadata.create_all(bind=engine) @@ -298,6 +303,57 @@ class ApiSmokeTests(unittest.TestCase): payload = response.json() return {"Authorization": f"Bearer {payload['access_token']}"}, payload + def _create_system_approver(self, headers: dict[str, str], *, tenant_id: str, email: str = "approver@example.local") -> dict[str, str]: + roles = self.client.get("/api/v1/admin/system/roles", headers=headers) + self.assertEqual(roles.status_code, 200, roles.text) + system_admin = next(item for item in roles.json()["roles"] if item["slug"] == "system_admin") + created = self.client.post( + "/api/v1/admin/system/accounts", + headers=headers, + json={ + "email": email, + "display_name": "Configuration Approver", + "password": "approver-password", + "password_reset_required": False, + "is_active": True, + "role_ids": [system_admin["id"]], + "memberships": [{ + "tenant_id": tenant_id, + "is_active": True, + "role_ids": [], + "group_ids": [], + }], + }, + ) + self.assertEqual(created.status_code, 201, created.text) + approver_headers, _ = self._login_as(email=email, password="approver-password") + return approver_headers + + def _approved_configuration_change( + self, + headers: dict[str, str], + approver_headers: dict[str, str], + *, + key: str, + value: object, + target: dict[str, object] | None = None, + ) -> str: + created = self.client.post( + "/api/v1/admin/configuration-change-requests", + headers=headers, + json={"key": key, "value": value, "dry_run": True, "target": target or {}, "reason": "test approval"}, + ) + self.assertEqual(created.status_code, 201, created.text) + request_id = created.json()["request"]["id"] + approved = self.client.post( + f"/api/v1/admin/configuration-change-requests/{request_id}/approve", + headers=approver_headers, + json={"reason": "approved by second system administrator"}, + ) + self.assertEqual(approved.status_code, 200, approved.text) + self.assertEqual(approved.json()["request"]["status"], "approved") + return request_id + def test_cookie_session_requires_csrf_for_mutations(self) -> None: response = self.client.post( @@ -310,6 +366,12 @@ class ApiSmokeTests(unittest.TestCase): me = self.client.get("/api/v1/auth/me") self.assertEqual(me.status_code, 200, me.text) + me_payload = me.json() + self.assertEqual(me_payload["principal"]["account_id"], me_payload["user"]["account_id"]) + self.assertEqual(me_payload["principal"]["membership_id"], me_payload["user"]["id"]) + self.assertEqual(me_payload["principal"]["tenant_id"], me_payload["tenant"]["id"]) + self.assertEqual(me_payload["principal"]["auth_method"], "session") + self.assertTrue(set(me_payload["principal"]["scopes"]).issuperset(set(me_payload["scopes"]))) blocked = self.client.patch("/api/v1/auth/profile", json={"display_name": "Blocked"}) self.assertEqual(blocked.status_code, 403, blocked.text) @@ -323,7 +385,7 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(allowed.json()["user"]["display_name"], "Cookie Admin") def test_mailbox_message_listing_reports_total_count(self) -> None: - headers, _ = self._login() + headers, login = self._login() created_profile = self.client.post( "/api/v1/mail/profiles", headers=headers, @@ -369,6 +431,113 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(payload["total_count"], 3) self.assertEqual(len(payload["messages"]), 2) self.assertTrue(all(message["folder"] == "INBOX" for message in payload["messages"])) + self.assertTrue(payload["cursor_stable"]) + self.assertFalse(payload["full"]) + self.assertTrue(payload["next_cursor"]) + + next_page = self.client.get( + f"/api/v1/mail/profiles/{profile_id}/mailbox/messages", + headers=headers, + params={"folder": "INBOX", "cursor": payload["next_cursor"]}, + ) + self.assertEqual(next_page.status_code, 200, next_page.text) + next_payload = next_page.json() + self.assertEqual(next_payload["offset"], 2) + self.assertEqual(len(next_payload["messages"]), 1) + self.assertFalse(next_payload["next_cursor"]) + + stale_cursor = encode_keyset_cursor( + "mail.mailbox.messages.v1", + fingerprint=keyset_query_fingerprint( + "mail.mailbox.messages.v1", + { + "tenant_id": login["tenant"]["id"], + "profile_id": profile_id, + "folder": "INBOX", + "limit": 2, + "sort": "imap.uid.desc", + }, + ), + values={ + "offset": 2, + "limit": 2, + "uidvalidity": "stale-mock-uidvalidity", + "after_uid": payload["messages"][-1]["uid"], + }, + ) + stale_page = self.client.get( + f"/api/v1/mail/profiles/{profile_id}/mailbox/messages", + headers=headers, + params={"folder": "INBOX", "cursor": stale_cursor}, + ) + self.assertEqual(stale_page.status_code, 200, stale_page.text) + stale_payload = stale_page.json() + self.assertTrue(stale_payload["full"]) + self.assertEqual(stale_payload["offset"], 0) + self.assertEqual(len(stale_payload["messages"]), 2) + self.assertTrue(stale_payload["next_cursor"]) + + mismatched_cursor = self.client.get( + f"/api/v1/mail/profiles/{profile_id}/mailbox/messages", + headers=headers, + params={"folder": "Sent", "cursor": payload["next_cursor"]}, + ) + self.assertEqual(mismatched_cursor.status_code, 400, mismatched_cursor.text) + + def test_mail_settings_delta_tracks_profiles_and_policy(self) -> None: + headers, _ = self._login() + full = self.client.get( + "/api/v1/mail/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_inactive": "true"}, + ) + self.assertEqual(full.status_code, 200, full.text) + full_payload = full.json() + self.assertTrue(full_payload["full"]) + + created_profile = self.client.post( + "/api/v1/mail/profiles", + headers=headers, + json={ + "name": "Delta mail settings", + "smtp": { + "host": "mock.smtp", + "port": 2525, + "security": "starttls", + }, + }, + ) + self.assertEqual(created_profile.status_code, 201, created_profile.text) + profile_id = created_profile.json()["id"] + + profile_delta = self.client.get( + "/api/v1/mail/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_inactive": "true", "since": full_payload["watermark"]}, + ) + self.assertEqual(profile_delta.status_code, 200, profile_delta.text) + profile_delta_payload = profile_delta.json() + self.assertFalse(profile_delta_payload["full"]) + self.assertIn("profiles", profile_delta_payload["changed_sections"]) + self.assertEqual({profile["id"] for profile in profile_delta_payload["profiles"]}, {profile_id}) + + updated_policy = self.client.put( + "/api/v1/mail/policies/tenant", + headers=headers, + json={"policy": {"allow_user_profiles": False}}, + ) + self.assertEqual(updated_policy.status_code, 200, updated_policy.text) + + policy_delta = self.client.get( + "/api/v1/mail/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_inactive": "true", "since": profile_delta_payload["watermark"]}, + ) + self.assertEqual(policy_delta.status_code, 200, policy_delta.text) + policy_delta_payload = policy_delta.json() + self.assertFalse(policy_delta_payload["full"]) + self.assertIn("policy", policy_delta_payload["changed_sections"]) + self.assertEqual(policy_delta_payload["policy"]["effective_policy"]["allow_user_profiles"], False) def test_encrypted_mail_profile_can_back_campaign_without_exposing_secrets(self) -> None: headers, _ = self._login() @@ -572,9 +741,121 @@ class ApiSmokeTests(unittest.TestCase): dev_mailbox = self.client.get("/api/v1/dev/mailbox/messages", headers=headers) self.assertEqual(dev_mailbox.status_code, 404, dev_mailbox.text) + def test_docs_context_and_ops_status_are_available(self) -> None: + headers, _ = self._login() + + docs = self.client.get("/api/v1/docs/context", headers=headers) + self.assertEqual(docs.status_code, 200, docs.text) + docs_payload = docs.json() + module_ids = {item["id"] for item in docs_payload["layers"]["configured"]["modules"]} + self.assertIn("docs", module_ids) + self.assertIn("ops", module_ids) + self.assertGreaterEqual(docs_payload["summary"]["visible_route_count"], 1) + self.assertGreaterEqual(docs_payload["summary"]["documentation_topic_count"], 1) + documentation_topic_ids = {item["id"] for item in docs_payload["layers"]["always"]["documentation"]} + self.assertIn("docs.configured-system-documentation", documentation_topic_ids) + workflow_topic_ids = {item["id"] for item in docs_payload["topic_groups"]["workflow"]} + reference_topic_ids = {item["id"] for item in docs_payload["topic_groups"]["reference"]} + pattern_topic_ids = {item["id"] for item in docs_payload["topic_groups"]["pattern"]} + self.assertIn("access.workflow.grant-user-access", workflow_topic_ids) + self.assertIn("access.reference.admin-access-fields", reference_topic_ids) + self.assertIn("docs.pattern.field-help", pattern_topic_ids) + workflow_topic = next(item for item in docs_payload["topic_groups"]["workflow"] if item["id"] == "access.workflow.grant-user-access") + self.assertEqual(workflow_topic["anchor_id"], "docs-topic-access-access-workflow-grant-user-access") + + user_docs = self.client.get("/api/v1/docs/context?type=user&locale=de", headers=headers) + self.assertEqual(user_docs.status_code, 200, user_docs.text) + user_docs_payload = user_docs.json() + self.assertEqual(user_docs_payload["actor"]["documentation_type"], "user") + self.assertEqual(user_docs_payload["actor"]["locale"], "de") + user_always = {item["id"]: item for item in user_docs_payload["layers"]["always"]["documentation"]} + self.assertEqual(user_always["docs.configured-system-documentation"]["translation_locale"], "de") + + platform_status = self.client.get("/api/v1/platform/status", headers=headers) + self.assertEqual(platform_status.status_code, 200, platform_status.text) + i18n_status = platform_status.json()["i18n"] + self.assertIn("en", i18n_status["enabled_languages"]) + self.assertIn("de", {item["code"] for item in i18n_status["available_languages"]}) + + ops = self.client.get("/api/v1/ops/status", headers=headers) + self.assertEqual(ops.status_code, 200, ops.text) + ops_payload = ops.json() + self.assertIn(ops_payload["summary"]["active_profile"], {"local-dev", "production-like-dev", "single-process", "split-worker"}) + self.assertEqual(ops_payload["summary"]["app_env"], "test") + self.assertIn("redis_url", ops_payload["summary"]) + self.assertIn("celery_queues", ops_payload["summary"]) + self.assertIn("readiness", ops_payload) + self.assertIn("module_registry", {item["id"] for item in ops_payload["checks"]}) + self.assertIn("redis_broker", {item["id"] for item in ops_payload["checks"]}) + self.assertIn("worker_split", {item["id"] for item in ops_payload["checks"]}) + self.assertIn("deployment_security", {item["id"] for item in ops_payload["checks"]}) + self.assertGreaterEqual(len(ops_payload["sizing"]), 1) + + readiness = self.client.get("/api/v1/ops/readiness", headers=headers) + self.assertEqual(readiness.status_code, 200, readiness.text) + self.assertTrue(readiness.json()["ready"]) + + def test_language_packages_tenant_enablement_and_user_preference(self) -> None: + headers, _ = self._login() + + settings_response = self.client.get("/api/v1/admin/system/settings", headers=headers) + self.assertEqual(settings_response.status_code, 200, settings_response.text) + settings_payload = settings_response.json() + installed_languages = settings_payload["available_languages"] + if "fr" not in {item["code"] for item in installed_languages}: + installed_languages = [ + *installed_languages, + {"code": "fr", "label": "French", "native_label": "Francais"}, + ] + enabled_codes = list(dict.fromkeys([*settings_payload["enabled_language_codes"], "fr"])) + system_update = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": settings_payload["default_locale"], + "allow_tenant_custom_groups": settings_payload["allow_tenant_custom_groups"], + "allow_tenant_custom_roles": settings_payload["allow_tenant_custom_roles"], + "allow_tenant_api_keys": settings_payload["allow_tenant_api_keys"], + "available_languages": installed_languages, + "enabled_language_codes": enabled_codes, + }, + ) + self.assertEqual(system_update.status_code, 200, system_update.text) + self.assertIn("fr", {item["code"] for item in system_update.json()["available_languages"]}) + self.assertIn("fr", system_update.json()["enabled_language_codes"]) + + platform_status = self.client.get("/api/v1/platform/status", headers=headers) + self.assertEqual(platform_status.status_code, 200, platform_status.text) + self.assertIn("fr", platform_status.json()["i18n"]["enabled_languages"]) + + tenant_update = self.client.patch( + "/api/v1/admin/tenant/settings", + headers=headers, + json={"default_locale": "fr", "enabled_language_codes": ["en", "fr"]}, + ) + self.assertEqual(tenant_update.status_code, 200, tenant_update.text) + self.assertEqual(tenant_update.json()["default_locale"], "fr") + self.assertEqual(tenant_update.json()["enabled_language_codes"], ["en", "fr"]) + + profile_update = self.client.patch( + "/api/v1/auth/profile", + headers=headers, + json={"preferred_language": "fr"}, + ) + self.assertEqual(profile_update.status_code, 200, profile_update.text) + profile_payload = profile_update.json() + self.assertEqual(profile_payload["user"]["preferred_language"], "fr") + self.assertEqual(profile_payload["default_language"], "fr") + self.assertEqual(profile_payload["enabled_language_codes"], ["en", "fr"]) + + refreshed_profile = self.client.get("/api/v1/auth/me", headers=headers) + self.assertEqual(refreshed_profile.status_code, 200, refreshed_profile.text) + self.assertEqual(refreshed_profile.json()["user"]["preferred_language"], "fr") + def test_managed_file_runtime_flow(self) -> None: headers, login = self._login() user_id = login["user"]["id"] + tenant_id = login["tenant"]["id"] spaces = self.client.get("/api/v1/files/spaces", headers=headers) self.assertEqual(spaces.status_code, 200, spaces.text) @@ -597,6 +878,82 @@ class ApiSmokeTests(unittest.TestCase): first_file = first.json()["files"][0] self.assertEqual(first_file["display_path"], "inbox/report.txt") self.assertEqual(first_file["content_type"], "text/plain") + self.assertIsNone(first_file["source_provenance"]) + self.assertIsNone(first_file["source_revision"]) + + connector_policy_sources = [ + { + "scope_type": "system", + "label": "System", + "policy": {"allow": {"providers": ["seafile"]}}, + }, + { + "scope_type": "tenant", + "scope_id": tenant_id, + "label": "Tenant", + "policy": {"deny": {"external_paths": ["/reports/private/*"]}}, + }, + ] + connector_policy_json = json.dumps({"sources": connector_policy_sources}) + allowed_source = { + "source_type": "connector", + "connector_id": "seafile-main", + "provider": "seafile", + "external_id": "repo-1:file-1", + "external_path": "/reports/imported.txt", + "metadata": {"library": "Reports"}, + } + denied_policy = self.client.post( + "/api/v1/files/connector-policy/evaluate", + headers=headers, + json={ + "operation": "import", + "source_provenance": {**allowed_source, "external_path": "/reports/private/secrets.txt"}, + "policy_sources": connector_policy_sources, + }, + ) + self.assertEqual(denied_policy.status_code, 200, denied_policy.text) + denied_decision = denied_policy.json()["decision"] + self.assertFalse(denied_decision["allowed"]) + self.assertIn("connector_policy_denylist", denied_decision["requirements"]) + self.assertEqual(denied_decision["details"]["deny_matches"][0]["scope"]["path"], f"tenant:{tenant_id}") + + blocked_upload = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={ + "owner_type": "user", + "owner_id": user_id, + "path": "inbox/blocked", + "source_revision": "nextcloud-rev-1", + "source_provenance_json": json.dumps({**allowed_source, "connector_id": "nextcloud-main", "provider": "nextcloud"}), + "connector_policy_json": connector_policy_json, + }, + files=[("files", ("blocked.txt", b"blocked", "text/plain"))], + ) + self.assertEqual(blocked_upload.status_code, 403, blocked_upload.text) + self.assertFalse(blocked_upload.json()["detail"]["allowed"]) + self.assertIn("connector_policy_allowlist", blocked_upload.json()["detail"]["requirements"]) + + imported = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={ + "owner_type": "user", + "owner_id": user_id, + "path": "inbox/imported", + "source_revision": "seafile-rev-1", + "source_provenance_json": json.dumps(allowed_source), + "connector_policy_json": connector_policy_json, + }, + files=[("files", ("imported.txt", b"imported report", "text/plain"))], + ) + self.assertEqual(imported.status_code, 200, imported.text) + imported_file = imported.json()["files"][0] + self.assertEqual(imported_file["source_revision"], "seafile-rev-1") + self.assertEqual(imported_file["source_provenance"]["connector_id"], "seafile-main") + self.assertEqual(imported_file["source_provenance"]["revision"], "seafile-rev-1") + self.assertEqual(imported_file["source_provenance"]["metadata"]["library"], "Reports") # Exercises request-model conversion to FileConflictResolution and the # explicit rename path rather than silently overwriting an asset. @@ -629,7 +986,29 @@ class ApiSmokeTests(unittest.TestCase): params={"owner_type": "user", "owner_id": user_id, "path_prefix": "inbox"}, ) self.assertEqual(listing.status_code, 200, listing.text) - self.assertEqual(len(listing.json()["files"]), 2) + self.assertEqual(len(listing.json()["files"]), 3) + listed_import = next(item for item in listing.json()["files"] if item["id"] == imported_file["id"]) + self.assertEqual(listed_import["source_revision"], "seafile-rev-1") + + downloaded_import = self.client.get(f"/api/v1/files/{imported_file['id']}/download", headers=headers) + self.assertEqual(downloaded_import.status_code, 200, downloaded_import.text) + self.assertEqual(downloaded_import.content, b"imported report") + + connector_audit = self.client.get( + "/api/v1/admin/audit", + headers=headers, + params={"limit": 500, "filter_action": "files.connector"}, + ) + self.assertEqual(connector_audit.status_code, 200, connector_audit.text) + connector_events = {item["action"]: item for item in connector_audit.json()["items"]} + imported_event = connector_events["files.connector.imported"] + accessed_event = connector_events["files.connector.accessed"] + self.assertEqual(imported_event["object_id"], imported_file["id"]) + self.assertEqual(imported_event["details"]["source_revision"], "seafile-rev-1") + self.assertEqual(imported_event["details"]["source_provenance"]["connector_id"], "seafile-main") + self.assertEqual(accessed_event["object_id"], imported_file["id"]) + self.assertEqual(accessed_event["details"]["operation"], "download") + self.assertEqual(accessed_event["details"]["source_revision"], "seafile-rev-1") resolved = self.client.post( "/api/v1/files/resolve-patterns", @@ -642,7 +1021,7 @@ class ApiSmokeTests(unittest.TestCase): }, ) self.assertEqual(resolved.status_code, 200, resolved.text) - self.assertEqual(len(resolved.json()["patterns"][0]["matches"]), 2) + self.assertEqual(len(resolved.json()["patterns"][0]["matches"]), 3) self.assertEqual(resolved.json()["unmatched"], []) # Exercises RenamePlanItem construction without mutating persisted paths. @@ -651,6 +1030,8 @@ class ApiSmokeTests(unittest.TestCase): headers=headers, json={ "file_ids": [first_file["id"]], + "owner_type": "user", + "owner_id": user_id, "mode": "prefix", "prefix": "archived-", "dry_run": True, @@ -659,6 +1040,59 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(rename_preview.status_code, 200, rename_preview.text) self.assertEqual(rename_preview.json()["items"][0]["new_path"], "inbox/archived-report.txt") + ownerless_rename = self.client.post( + "/api/v1/files/bulk-rename", + headers=headers, + json={"file_ids": [first_file["id"]], "mode": "prefix", "prefix": "archived-", "dry_run": True}, + ) + self.assertEqual(ownerless_rename.status_code, 422, ownerless_rename.text) + + legacy_role = self._create_role(headers, slug="legacy-file-organizer", permissions=["files:organize"]) + legacy_user = self._create_user( + headers, + email="legacy-file-organizer@example.local", + password="legacy-file-organizer-password", + role_ids=[str(legacy_role["id"])], + ) + legacy_read_share = self.client.post( + f"/api/v1/files/{first_file['id']}/shares", + headers=headers, + json={"target_type": "user", "target_id": legacy_user["id"], "permission": "read"}, + ) + self.assertEqual(legacy_read_share.status_code, 200, legacy_read_share.text) + + from govoplan_files.backend.db.models import FileShare + from govoplan_files.backend.storage.common import FileStorageError + from govoplan_files.backend.storage.transfers import legacy_rename_selection_without_owner_context + + with SessionLocal() as session: + with self.assertRaises(FileStorageError): + legacy_rename_selection_without_owner_context( + session, + tenant_id=str(login["tenant"]["id"]), + user_id=str(legacy_user["id"]), + file_ids=[first_file["id"]], + mode="prefix", + prefix="blocked-", + dry_run=True, + ) + share = session.query(FileShare).filter(FileShare.id == legacy_read_share.json()["id"]).one() + share.permission = "write" + session.add(share) + session.commit() + + with SessionLocal() as session: + legacy_plan = legacy_rename_selection_without_owner_context( + session, + tenant_id=str(login["tenant"]["id"]), + user_id=str(legacy_user["id"]), + file_ids=[first_file["id"]], + mode="prefix", + prefix="shared-", + dry_run=True, + ) + self.assertEqual(legacy_plan[0].new_path, "inbox/shared-report.txt") + invalid_share = self.client.post( f"/api/v1/files/{first_file['id']}/shares", headers=headers, @@ -762,6 +1196,1446 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(transferred.status_code, 200, transferred.text) self.assertEqual(transferred.json()["files"], 1) + def test_files_delta_tracks_uploads_folders_and_delete_tombstones(self) -> None: + headers, login = self._login() + user_id = login["user"]["id"] + + full = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={"owner_type": "user", "owner_id": user_id}, + ) + self.assertEqual(full.status_code, 200, full.text) + self.assertTrue(full.json()["full"]) + watermark = full.json()["watermark"] + + folder = self.client.post( + "/api/v1/files/folders", + headers=headers, + json={"owner_type": "user", "owner_id": user_id, "path": "delta"}, + ) + self.assertEqual(folder.status_code, 200, folder.text) + uploaded = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={"owner_type": "user", "owner_id": user_id, "path": "delta"}, + files=[("files", ("delta.txt", b"delta payload", "text/plain"))], + ) + self.assertEqual(uploaded.status_code, 200, uploaded.text) + file_id = uploaded.json()["files"][0]["id"] + + delta = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "since": watermark}, + ) + self.assertEqual(delta.status_code, 200, delta.text) + delta_payload = delta.json() + self.assertFalse(delta_payload["full"]) + self.assertEqual({item["id"] for item in delta_payload["files"]}, {file_id}) + self.assertEqual({item["path"] for item in delta_payload["folders"]}, {"delta"}) + self.assertEqual(delta_payload["deleted"], []) + + deleted = self.client.delete(f"/api/v1/files/{file_id}", headers=headers) + self.assertEqual(deleted.status_code, 200, deleted.text) + after_delete = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "since": delta_payload["watermark"]}, + ) + self.assertEqual(after_delete.status_code, 200, after_delete.text) + deleted_items = after_delete.json()["deleted"] + self.assertEqual(len(deleted_items), 1) + self.assertEqual(deleted_items[0]["id"], file_id) + self.assertEqual(deleted_items[0]["resource_type"], "file") + self.assertTrue(str(deleted_items[0]["revision"]).startswith("seq:")) + + def test_files_and_folders_support_cursor_windows(self) -> None: + headers, login = self._login() + user_id = login["user"]["id"] + + for folder_path in ("cursor-a", "cursor-b", "cursor-c"): + folder = self.client.post( + "/api/v1/files/folders", + headers=headers, + json={"owner_type": "user", "owner_id": user_id, "path": folder_path}, + ) + self.assertEqual(folder.status_code, 200, folder.text) + + uploaded = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={"owner_type": "user", "owner_id": user_id, "path": "cursor-a"}, + files=[ + ("files", ("a.txt", b"a", "text/plain")), + ("files", ("b.txt", b"b", "text/plain")), + ("files", ("c.txt", b"c", "text/plain")), + ], + ) + self.assertEqual(uploaded.status_code, 200, uploaded.text) + + first_files = self.client.get( + "/api/v1/files", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "page_size": 2}, + ) + self.assertEqual(first_files.status_code, 200, first_files.text) + first_files_payload = first_files.json() + self.assertEqual(len(first_files_payload["files"]), 2) + self.assertTrue(first_files_payload["next_cursor"]) + + second_files = self.client.get( + "/api/v1/files", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "cursor": first_files_payload["next_cursor"]}, + ) + self.assertEqual(second_files.status_code, 200, second_files.text) + self.assertEqual(len(second_files.json()["files"]), 1) + + mismatched_files = self.client.get( + "/api/v1/files", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "path_prefix": "cursor-b", "cursor": first_files_payload["next_cursor"]}, + ) + self.assertEqual(mismatched_files.status_code, 400, mismatched_files.text) + + first_folders = self.client.get( + "/api/v1/files/folders", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "page_size": 2}, + ) + self.assertEqual(first_folders.status_code, 200, first_folders.text) + first_folders_payload = first_folders.json() + self.assertEqual(len(first_folders_payload["folders"]), 2) + self.assertTrue(first_folders_payload["next_cursor"]) + + second_folders = self.client.get( + "/api/v1/files/folders", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "cursor": first_folders_payload["next_cursor"]}, + ) + self.assertEqual(second_folders.status_code, 200, second_folders.text) + self.assertEqual(len(second_folders.json()["folders"]), 1) + + mismatched_folders = self.client.get( + "/api/v1/files/folders", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "page_size": 3, "cursor": first_folders_payload["next_cursor"]}, + ) + self.assertEqual(mismatched_folders.status_code, 400, mismatched_folders.text) + + def test_files_delta_reports_active_window_tombstones_for_moves_and_folder_deletes(self) -> None: + headers, login = self._login() + user_id = login["user"]["id"] + + source_folder = self.client.post( + "/api/v1/files/folders", + headers=headers, + json={"owner_type": "user", "owner_id": user_id, "path": "window-source"}, + ) + self.assertEqual(source_folder.status_code, 200, source_folder.text) + uploaded = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={"owner_type": "user", "owner_id": user_id, "path": "window-source"}, + files=[("files", ("moving.txt", b"moving", "text/plain"))], + ) + self.assertEqual(uploaded.status_code, 200, uploaded.text) + moving_file_id = uploaded.json()["files"][0]["id"] + + source_window = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "path_prefix": "window-source"}, + ) + self.assertEqual(source_window.status_code, 200, source_window.text) + moved = self.client.post( + "/api/v1/files/transfer", + headers=headers, + json={ + "operation": "move", + "file_ids": [moving_file_id], + "folder_paths": [], + "source_owner_type": "user", + "source_owner_id": user_id, + "target_owner_type": "user", + "target_owner_id": user_id, + "target_folder": "window-target", + "conflict_strategy": "reject", + }, + ) + self.assertEqual(moved.status_code, 200, moved.text) + + after_move = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={ + "owner_type": "user", + "owner_id": user_id, + "path_prefix": "window-source", + "since": source_window.json()["watermark"], + }, + ) + self.assertEqual(after_move.status_code, 200, after_move.text) + move_deleted = after_move.json()["deleted"] + self.assertIn(moving_file_id, {item["id"] for item in move_deleted if item["resource_type"] == "file"}) + self.assertEqual(after_move.json()["files"], []) + + delete_root = self.client.post( + "/api/v1/files/folders", + headers=headers, + json={"owner_type": "user", "owner_id": user_id, "path": "window-delete"}, + ) + self.assertEqual(delete_root.status_code, 200, delete_root.text) + nested = self.client.post( + "/api/v1/files/folders", + headers=headers, + json={"owner_type": "user", "owner_id": user_id, "path": "window-delete/nested"}, + ) + self.assertEqual(nested.status_code, 200, nested.text) + uploaded_nested = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={"owner_type": "user", "owner_id": user_id, "path": "window-delete/nested"}, + files=[("files", ("deleted.txt", b"deleted", "text/plain"))], + ) + self.assertEqual(uploaded_nested.status_code, 200, uploaded_nested.text) + nested_file_id = uploaded_nested.json()["files"][0]["id"] + delete_window = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={"owner_type": "user", "owner_id": user_id, "path_prefix": "window-delete"}, + ) + self.assertEqual(delete_window.status_code, 200, delete_window.text) + + deleted_folder = self.client.post( + "/api/v1/files/folders/delete", + headers=headers, + json={"owner_type": "user", "owner_id": user_id, "path": "window-delete", "recursive": True}, + ) + self.assertEqual(deleted_folder.status_code, 200, deleted_folder.text) + after_folder_delete = self.client.get( + "/api/v1/files/delta", + headers=headers, + params={ + "owner_type": "user", + "owner_id": user_id, + "path_prefix": "window-delete", + "since": delete_window.json()["watermark"], + }, + ) + self.assertEqual(after_folder_delete.status_code, 200, after_folder_delete.text) + folder_delete_items = after_folder_delete.json()["deleted"] + self.assertIn(delete_root.json()["id"], {item["id"] for item in folder_delete_items if item["resource_type"] == "folder"}) + self.assertIn(nested_file_id, {item["id"] for item in folder_delete_items if item["resource_type"] == "file"}) + + def test_files_delta_omits_private_tombstones_from_broad_reader_feed(self) -> None: + owner_headers, owner_login = self._login() + owner_id = owner_login["user"]["id"] + + uploaded = self.client.post( + "/api/v1/files/upload", + headers=owner_headers, + data={"owner_type": "user", "owner_id": owner_id, "path": "private"}, + files=[("files", ("private-delta.txt", b"private delta", "text/plain"))], + ) + self.assertEqual(uploaded.status_code, 200, uploaded.text) + file_id = uploaded.json()["files"][0]["id"] + + reader_role = self._create_role( + owner_headers, + slug="delta-broad-reader", + permissions=["files:read"], + ) + self._create_user( + owner_headers, + email="delta-broad-reader@example.local", + password="delta-broad-reader-password", + role_ids=[str(reader_role["id"])], + ) + reader_headers, _ = self._login_as( + email="delta-broad-reader@example.local", + password="delta-broad-reader-password", + ) + + full = self.client.get("/api/v1/files/delta", headers=reader_headers) + self.assertEqual(full.status_code, 200, full.text) + full_payload = full.json() + self.assertTrue(full_payload["full"]) + self.assertNotIn(file_id, {item["id"] for item in full_payload["files"]}) + + deleted = self.client.delete(f"/api/v1/files/{file_id}", headers=owner_headers) + self.assertEqual(deleted.status_code, 200, deleted.text) + + incremental = self.client.get( + "/api/v1/files/delta", + headers=reader_headers, + params={"since": full_payload["watermark"]}, + ) + self.assertEqual(incremental.status_code, 200, incremental.text) + incremental_payload = incremental.json() + self.assertFalse(incremental_payload["full"]) + self.assertNotIn(file_id, {item["id"] for item in incremental_payload["files"]}) + self.assertNotIn(file_id, {item["id"] for item in incremental_payload["deleted"]}) + + def test_calendar_event_window_delta_tracks_moves_deletes_and_stale_watermarks(self) -> None: + headers, _ = self._login() + calendars = self.client.get("/api/v1/calendar/calendars", headers=headers) + self.assertEqual(calendars.status_code, 200, calendars.text) + if calendars.json()["calendars"]: + calendar_id = calendars.json()["calendars"][0]["id"] + else: + created_calendar = self.client.post( + "/api/v1/calendar/calendars", + headers=headers, + json={"name": "Delta calendar", "slug": "delta-calendar", "is_default": True}, + ) + self.assertEqual(created_calendar.status_code, 201, created_calendar.text) + calendar_id = created_calendar.json()["id"] + + window_start = "2030-01-01T00:00:00+00:00" + window_end = "2030-01-31T23:59:59+00:00" + initial = self.client.get( + "/api/v1/calendar/events/delta", + headers=headers, + params={"calendar_id": calendar_id, "start_at": window_start, "end_at": window_end}, + ) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + + created = self.client.post( + "/api/v1/calendar/events", + headers=headers, + json={ + "calendar_id": calendar_id, + "summary": "Window delta", + "start_at": "2030-01-10T10:00:00+00:00", + "end_at": "2030-01-10T11:00:00+00:00", + }, + ) + self.assertEqual(created.status_code, 201, created.text) + event_id = created.json()["id"] + + created_delta = self.client.get( + "/api/v1/calendar/events/delta", + headers=headers, + params={"calendar_id": calendar_id, "start_at": window_start, "end_at": window_end, "since": initial_payload["watermark"]}, + ) + self.assertEqual(created_delta.status_code, 200, created_delta.text) + created_delta_payload = created_delta.json() + self.assertFalse(created_delta_payload["full"]) + self.assertIn(event_id, {item["id"] for item in created_delta_payload["events"]}) + + moved = self.client.patch( + f"/api/v1/calendar/events/{event_id}", + headers=headers, + json={"start_at": "2030-03-01T10:00:00+00:00", "end_at": "2030-03-01T11:00:00+00:00"}, + ) + self.assertEqual(moved.status_code, 200, moved.text) + moved_delta = self.client.get( + "/api/v1/calendar/events/delta", + headers=headers, + params={"calendar_id": calendar_id, "start_at": window_start, "end_at": window_end, "since": created_delta_payload["watermark"]}, + ) + self.assertEqual(moved_delta.status_code, 200, moved_delta.text) + moved_delta_payload = moved_delta.json() + self.assertFalse(moved_delta_payload["full"]) + self.assertTrue(any(item["id"] == event_id and item["resource_type"] == "calendar_event" for item in moved_delta_payload["deleted"])) + self.assertNotIn(event_id, {item["id"] for item in moved_delta_payload["events"]}) + + visible = self.client.post( + "/api/v1/calendar/events", + headers=headers, + json={ + "calendar_id": calendar_id, + "summary": "Delete delta", + "start_at": "2030-01-12T10:00:00+00:00", + "end_at": "2030-01-12T11:00:00+00:00", + }, + ) + self.assertEqual(visible.status_code, 201, visible.text) + visible_id = visible.json()["id"] + visible_delta = self.client.get( + "/api/v1/calendar/events/delta", + headers=headers, + params={"calendar_id": calendar_id, "start_at": window_start, "end_at": window_end, "since": moved_delta_payload["watermark"]}, + ) + self.assertEqual(visible_delta.status_code, 200, visible_delta.text) + self.assertIn(visible_id, {item["id"] for item in visible_delta.json()["events"]}) + + deleted = self.client.delete(f"/api/v1/calendar/events/{visible_id}", headers=headers) + self.assertEqual(deleted.status_code, 204, deleted.text) + deleted_delta = self.client.get( + "/api/v1/calendar/events/delta", + headers=headers, + params={"calendar_id": calendar_id, "start_at": window_start, "end_at": window_end, "since": visible_delta.json()["watermark"]}, + ) + self.assertEqual(deleted_delta.status_code, 200, deleted_delta.text) + deleted_delta_payload = deleted_delta.json() + self.assertTrue(any(item["id"] == visible_id and item["resource_type"] == "calendar_event" for item in deleted_delta_payload["deleted"])) + + with SessionLocal() as session: + prune_sequence_entries( + session, + before_or_equal_id=decode_sequence_watermark(deleted_delta_payload["watermark"]), + tenant_id=created.json()["tenant_id"], + module_id="calendar", + collections=("calendar.events",), + ) + session.commit() + + stale = self.client.get( + "/api/v1/calendar/events/delta", + headers=headers, + params={"calendar_id": calendar_id, "start_at": window_start, "end_at": window_end, "since": initial_payload["watermark"]}, + ) + self.assertEqual(stale.status_code, 200, stale.text) + self.assertTrue(stale.json()["full"]) + + def test_file_connector_settings_delta_tracks_credentials_profiles_and_policy(self) -> None: + headers, _login = self._login() + full = self.client.get( + "/api/v1/files/connectors/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_disabled": "true"}, + ) + self.assertEqual(full.status_code, 200, full.text) + full_payload = full.json() + self.assertTrue(full_payload["full"]) + + credential = self.client.post( + "/api/v1/files/connectors/credentials", + headers=headers, + json={ + "id": "delta-webdav-credentials", + "label": "Delta WebDAV Credentials", + "provider": "webdav", + "scope_type": "tenant", + "credential_mode": "basic", + "credentials": {"username": "govoplan", "password": "delta-secret"}, + }, + ) + self.assertEqual(credential.status_code, 201, credential.text) + + credential_delta = self.client.get( + "/api/v1/files/connectors/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_disabled": "true", "since": full_payload["watermark"]}, + ) + self.assertEqual(credential_delta.status_code, 200, credential_delta.text) + credential_delta_payload = credential_delta.json() + self.assertFalse(credential_delta_payload["full"]) + self.assertIn("credentials", credential_delta_payload["changed_sections"]) + self.assertEqual({item["id"] for item in credential_delta_payload["credentials"]}, {"delta-webdav-credentials"}) + + profile = self.client.post( + "/api/v1/files/connectors/profiles", + headers=headers, + json={ + "id": "delta-webdav", + "label": "Delta WebDAV", + "provider": "webdav", + "endpoint_url": "http://127.0.0.1:9083", + "scope_type": "tenant", + "credential_profile_id": "delta-webdav-credentials", + "capabilities": ["browse"], + }, + ) + self.assertEqual(profile.status_code, 201, profile.text) + + profile_delta = self.client.get( + "/api/v1/files/connectors/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_disabled": "true", "since": credential_delta_payload["watermark"]}, + ) + self.assertEqual(profile_delta.status_code, 200, profile_delta.text) + profile_delta_payload = profile_delta.json() + self.assertFalse(profile_delta_payload["full"]) + self.assertIn("profiles", profile_delta_payload["changed_sections"]) + self.assertEqual({item["id"] for item in profile_delta_payload["profiles"]}, {"delta-webdav"}) + + updated_credential = self.client.patch( + "/api/v1/files/connectors/credentials/delta-webdav-credentials", + headers=headers, + json={"label": "Updated Delta WebDAV Credentials"}, + ) + self.assertEqual(updated_credential.status_code, 200, updated_credential.text) + + dependency_delta = self.client.get( + "/api/v1/files/connectors/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_disabled": "true", "since": profile_delta_payload["watermark"]}, + ) + self.assertEqual(dependency_delta.status_code, 200, dependency_delta.text) + dependency_delta_payload = dependency_delta.json() + self.assertIn("credentials", dependency_delta_payload["changed_sections"]) + self.assertIn("profiles", dependency_delta_payload["changed_sections"]) + self.assertEqual(dependency_delta_payload["profiles"][0]["credential_profile_label"], "Updated Delta WebDAV Credentials") + + policy = self.client.put( + "/api/v1/files/connectors/policies/tenant", + headers=headers, + json={"policy": {"deny": {"providers": ["nfs"]}}}, + ) + self.assertEqual(policy.status_code, 200, policy.text) + policy_delta = self.client.get( + "/api/v1/files/connectors/settings/delta", + headers=headers, + params={"scope_type": "tenant", "include_disabled": "true", "since": dependency_delta_payload["watermark"]}, + ) + self.assertEqual(policy_delta.status_code, 200, policy_delta.text) + policy_delta_payload = policy_delta.json() + self.assertFalse(policy_delta_payload["full"]) + self.assertIn("policy", policy_delta_payload["changed_sections"]) + self.assertEqual(policy_delta_payload["policy"]["effective_policy"]["deny"]["providers"], ["nfs"]) + + def test_file_connector_profiles_are_governed_and_redacted(self) -> None: + headers, login = self._login() + user_id = str(login["user"]["id"]) + tenant_id = str(login["tenant"]["id"]) + env_keys = [ + "GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", + "GOVOPLAN_TEST_SEAFILE_PASSWORD", + "GOVOPLAN_TEST_NEXTCLOUD_TOKEN", + "GOVOPLAN_TEST_SMB_PASSWORD", + ] + previous_env = {key: os.environ.get(key) for key in env_keys} + try: + os.environ["GOVOPLAN_TEST_SEAFILE_PASSWORD"] = "super-secret-seafile" + os.environ["GOVOPLAN_TEST_NEXTCLOUD_TOKEN"] = "super-secret-nextcloud" + os.environ["GOVOPLAN_TEST_SMB_PASSWORD"] = "super-secret-smb" + os.environ["GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON"] = json.dumps( + { + "profiles": [ + { + "id": "seafile-dev", + "label": "Seafile Dev", + "provider": "seafile", + "endpoint_url": "http://127.0.0.1:9082", + "scope_type": "system", + "credential_mode": "basic", + "username": "admin@example.local", + "password_env": "GOVOPLAN_TEST_SEAFILE_PASSWORD", + "capabilities": ["browse", "import"], + "metadata": { + "static_listing": { + "": [ + {"kind": "library", "name": "Reports", "path": "reports", "external_id": "lib-reports"}, + {"kind": "folder", "name": "Inbox", "path": "inbox"}, + {"kind": "file", "name": "readme.txt", "path": "readme.txt", "size_bytes": 12, "content_type": "text/plain"}, + ], + "inbox": [ + {"kind": "file", "name": "imported.txt", "path": "inbox/imported.txt", "size_bytes": 15, "etag": "etag-1"} + ], + } + }, + "policy": {"allow": {"providers": ["seafile"]}, "deny": {"external_paths": ["blocked"]}}, + }, + { + "id": "tenant-webdav", + "provider": "webdav", + "base_url": "http://127.0.0.1:9083", + "scope_type": "tenant", + "scope_id": tenant_id, + "credentials": { + "type": "basic", + "username": "govoplan", + "password_env": "GOVOPLAN_TEST_WEBDAV_PASSWORD", + }, + }, + { + "id": "user-nextcloud", + "provider": "nextcloud", + "url": "http://127.0.0.1:9081", + "scope_type": "user", + "scope_id": user_id, + "credential_mode": "token", + "token_env": "GOVOPLAN_TEST_NEXTCLOUD_TOKEN", + }, + { + "id": "tenant-smb", + "provider": "smb", + "endpoint_url": "smb://127.0.0.1:1445/files/root", + "scope_type": "tenant", + "scope_id": tenant_id, + "credential_mode": "basic", + "username": "govoplan", + "password_env": "GOVOPLAN_TEST_SMB_PASSWORD", + "capabilities": ["browse", "import"], + "policy": {"allow": {"providers": ["smb"]}}, + }, + { + "id": "other-tenant-smb", + "provider": "smb", + "endpoint_url": "smb://127.0.0.1:1445/files", + "scope_type": "tenant", + "scope_id": "other-tenant", + "credential_mode": "secret_ref", + "secret_ref": "vault://govoplan/files/smb", + }, + { + "id": "disabled-seafile", + "provider": "seafile", + "endpoint_url": "http://127.0.0.1:9082", + "enabled": False, + }, + ] + } + ) + + provider_response = self.client.get("/api/v1/files/connectors/providers", headers=headers) + self.assertEqual(provider_response.status_code, 200, provider_response.text) + providers = {item["provider"]: item for item in provider_response.json()["providers"]} + self.assertTrue({"seafile", "nextcloud", "webdav", "smb", "nfs", "local"}.issubset(providers)) + self.assertTrue(providers["seafile"]["implemented"]) + self.assertTrue(providers["seafile"]["browse_supported"]) + self.assertTrue(providers["seafile"]["import_supported"]) + self.assertEqual(providers["smb"]["optional_dependency"], "smbprotocol") + self.assertTrue(providers["smb"]["implemented"]) + self.assertTrue(providers["smb"]["browse_supported"]) + self.assertTrue(providers["smb"]["import_supported"]) + self.assertIn("files.connector.imported", providers["nextcloud"]["audit_events"]) + self.assertIn("files.connector.synced", providers["nextcloud"]["audit_events"]) + + response = self.client.get("/api/v1/files/connectors/profiles", headers=headers) + self.assertEqual(response.status_code, 200, response.text) + profiles = {item["id"]: item for item in response.json()["profiles"]} + self.assertEqual(set(profiles), {"seafile-dev", "tenant-webdav", "user-nextcloud", "tenant-smb"}) + self.assertTrue(profiles["seafile-dev"]["credentials_configured"]) + self.assertEqual(profiles["seafile-dev"]["credential_source"], "password_env") + self.assertEqual(profiles["seafile-dev"]["source_path"], "system") + self.assertEqual(profiles["seafile-dev"]["source_kind"], "settings") + self.assertEqual(profiles["tenant-webdav"]["source_path"], f"tenant:{tenant_id}") + self.assertFalse(profiles["tenant-webdav"]["credentials_configured"]) + self.assertEqual(profiles["user-nextcloud"]["source_path"], f"user:{user_id}") + self.assertEqual(profiles["seafile-dev"]["policy_sources"][0]["policy"]["allow"]["providers"], ["seafile"]) + redacted_payload = json.dumps(response.json()).lower() + self.assertNotIn("super-secret", redacted_payload) + self.assertNotIn("govoplan_test_", redacted_payload) + self.assertNotIn("secret_ref", redacted_payload) + + system_policy = self.client.get("/api/v1/files/connectors/policies/system", headers=headers) + self.assertEqual(system_policy.status_code, 200, system_policy.text) + self.assertEqual(system_policy.json()["scope_type"], "system") + self.assertEqual(system_policy.json()["scope_id"], None) + self.assertEqual(system_policy.json()["effective_policy_sources"], []) + + tenant_policy = self.client.get("/api/v1/files/connectors/policies/tenant", headers=headers) + self.assertEqual(tenant_policy.status_code, 200, tenant_policy.text) + self.assertEqual(tenant_policy.json()["scope_type"], "tenant") + self.assertEqual(tenant_policy.json()["scope_id"], tenant_id) + self.assertEqual(tenant_policy.json()["parent_policy_sources"], []) + + locked_system_policy = self.client.put( + "/api/v1/files/connectors/policies/system", + headers=headers, + json={"policy": {"allow_lower_level_limits": {"deny.credentials": False}}}, + ) + self.assertEqual(locked_system_policy.status_code, 200, locked_system_policy.text) + locked_tenant_policy = self.client.put( + "/api/v1/files/connectors/policies/tenant", + headers=headers, + json={"policy": {"deny": {"credentials": ["forbidden-by-parent"]}}}, + ) + self.assertEqual(locked_tenant_policy.status_code, 400, locked_tenant_policy.text) + reset_system_policy = self.client.put("/api/v1/files/connectors/policies/system", headers=headers, json={"policy": {}}) + self.assertEqual(reset_system_policy.status_code, 200, reset_system_policy.text) + + tenant_config_policy = self.client.put( + "/api/v1/files/connectors/policies/tenant", + headers=headers, + json={"policy": {"deny": {"providers": ["nfs"]}}}, + ) + self.assertEqual(tenant_config_policy.status_code, 200, tenant_config_policy.text) + self.assertEqual(tenant_config_policy.json()["policy"]["deny"]["providers"], ["nfs"]) + denied_profile_create = self.client.post( + "/api/v1/files/connectors/profiles", + headers=headers, + json={ + "id": "denied-nfs", + "label": "Denied NFS", + "provider": "nfs", + "endpoint_url": "nfs://127.0.0.1/export", + "scope_type": "tenant", + }, + ) + self.assertEqual(denied_profile_create.status_code, 403, denied_profile_create.text) + self.assertIn("connector_policy_denylist", denied_profile_create.json()["detail"]["requirements"]) + + central_blocked_credential = self.client.post( + "/api/v1/files/connectors/credentials", + headers=headers, + json={ + "id": "central-blocked-webdav-credentials", + "label": "Central Blocked WebDAV Credentials", + "provider": "webdav", + "scope_type": "tenant", + "credential_mode": "basic", + "credentials": {"username": "govoplan", "password": "central-secret-webdav"}, + }, + ) + self.assertEqual(central_blocked_credential.status_code, 201, central_blocked_credential.text) + central_blocked_profile = self.client.post( + "/api/v1/files/connectors/profiles", + headers=headers, + json={ + "id": "central-blocked-webdav", + "label": "Central Blocked WebDAV", + "provider": "webdav", + "endpoint_url": "http://127.0.0.1:9083", + "scope_type": "tenant", + "credential_profile_id": "central-blocked-webdav-credentials", + "capabilities": ["browse"], + "metadata": {"static_listing": {"": [{"kind": "file", "name": "central-blocked.txt", "path": "central-blocked.txt"}]}}, + }, + ) + self.assertEqual(central_blocked_profile.status_code, 201, central_blocked_profile.text) + central_runtime_policy = self.client.put( + "/api/v1/files/connectors/policies/tenant", + headers=headers, + json={"policy": {"deny": {"credentials": ["central-blocked-webdav-credentials"]}}}, + ) + self.assertEqual(central_runtime_policy.status_code, 200, central_runtime_policy.text) + self.assertEqual(central_runtime_policy.json()["effective_policy"]["deny"]["credentials"], ["central-blocked-webdav-credentials"]) + self.assertEqual(central_runtime_policy.json()["effective_policy_sources"][0]["label"], "Tenant connector policy") + central_blocked_browse = self.client.get("/api/v1/files/connectors/profiles/central-blocked-webdav/browse", headers=headers) + self.assertEqual(central_blocked_browse.status_code, 403, central_blocked_browse.text) + self.assertIn("connector_policy_denylist", central_blocked_browse.json()["detail"]["requirements"]) + self.assertEqual(central_blocked_browse.json()["detail"]["source_path"][0]["label"], "Tenant connector policy") + + stored_credential = self.client.post( + "/api/v1/files/connectors/credentials", + headers=headers, + json={ + "id": "stored-webdav-credentials", + "label": "Stored WebDAV Credentials", + "provider": "webdav", + "scope_type": "tenant", + "credential_mode": "basic", + "credentials": {"username": "govoplan", "password": "stored-secret-webdav"}, + "policy": {"allow": {"credentials": ["stored-webdav-credentials"], "providers": ["webdav"]}}, + }, + ) + self.assertEqual(stored_credential.status_code, 201, stored_credential.text) + stored_credential_payload = stored_credential.json() + self.assertEqual(stored_credential_payload["credential_secret_source"], "stored") + self.assertTrue(stored_credential_payload["credentials_configured"]) + self.assertNotIn("stored-secret-webdav", json.dumps(stored_credential_payload)) + + stored_profile = self.client.post( + "/api/v1/files/connectors/profiles", + headers=headers, + json={ + "id": "stored-webdav", + "label": "Stored WebDAV", + "provider": "webdav", + "endpoint_url": "http://127.0.0.1:9083", + "scope_type": "tenant", + "credential_profile_id": "stored-webdav-credentials", + "credential_mode": "basic", + "capabilities": ["browse", "import", "sync"], + "policy": {"allow": {"providers": ["webdav"]}}, + "metadata": { + "static_listing": { + "": [ + {"kind": "folder", "name": "GovOPlaN", "path": "GovOPlaN"}, + {"kind": "file", "name": "notice.txt", "path": "notice.txt"}, + ] + } + }, + }, + ) + self.assertEqual(stored_profile.status_code, 201, stored_profile.text) + stored_payload = stored_profile.json() + self.assertEqual(stored_payload["source_kind"], "database") + self.assertEqual(stored_payload["credential_source"], "credential_profile") + self.assertEqual(stored_payload["credential_profile_id"], "stored-webdav-credentials") + self.assertEqual(stored_payload["credential_profile_label"], "Stored WebDAV Credentials") + self.assertTrue(stored_payload["credentials_configured"]) + self.assertNotIn("stored-secret-webdav", json.dumps(stored_payload)) + + stored_browse = self.client.get("/api/v1/files/connectors/profiles/stored-webdav/browse", headers=headers) + self.assertEqual(stored_browse.status_code, 200, stored_browse.text) + self.assertEqual([item["path"] for item in stored_browse.json()["items"]], ["GovOPlaN", "notice.txt"]) + + stored_list = self.client.get("/api/v1/files/connectors/profiles?include_disabled=true", headers=headers) + self.assertEqual(stored_list.status_code, 200, stored_list.text) + self.assertIn("stored-webdav", {item["id"] for item in stored_list.json()["profiles"]}) + + disabled_stored = self.client.delete("/api/v1/files/connectors/profiles/stored-webdav", headers=headers) + self.assertEqual(disabled_stored.status_code, 200, disabled_stored.text) + self.assertFalse(disabled_stored.json()["enabled"]) + + blocked_credential = self.client.post( + "/api/v1/files/connectors/credentials", + headers=headers, + json={ + "id": "blocked-webdav-credentials", + "label": "Blocked WebDAV Credentials", + "provider": "webdav", + "scope_type": "tenant", + "credential_mode": "basic", + "credentials": {"username": "govoplan", "password": "blocked-secret-webdav"}, + "policy": {"deny": {"credentials": ["blocked-webdav-credentials"]}}, + }, + ) + self.assertEqual(blocked_credential.status_code, 201, blocked_credential.text) + blocked_credential_profile = self.client.post( + "/api/v1/files/connectors/profiles", + headers=headers, + json={ + "id": "blocked-credential-webdav", + "label": "Blocked Credential WebDAV", + "provider": "webdav", + "endpoint_url": "http://127.0.0.1:9083", + "scope_type": "tenant", + "credential_profile_id": "blocked-webdav-credentials", + "capabilities": ["browse"], + "metadata": {"static_listing": {"": [{"kind": "file", "name": "blocked.txt", "path": "blocked.txt"}]}}, + }, + ) + self.assertEqual(blocked_credential_profile.status_code, 201, blocked_credential_profile.text) + blocked_credential_browse = self.client.get("/api/v1/files/connectors/profiles/blocked-credential-webdav/browse", headers=headers) + self.assertEqual(blocked_credential_browse.status_code, 403, blocked_credential_browse.text) + self.assertIn("connector_policy_denylist", blocked_credential_browse.json()["detail"]["requirements"]) + + browse_root = self.client.get("/api/v1/files/connectors/profiles/seafile-dev/browse", headers=headers) + self.assertEqual(browse_root.status_code, 200, browse_root.text) + self.assertTrue(browse_root.json()["read_only"]) + self.assertTrue(browse_root.json()["decision"]["allowed"]) + self.assertEqual([item["kind"] for item in browse_root.json()["items"]], ["library", "folder", "file"]) + self.assertEqual([item["name"] for item in browse_root.json()["items"]], ["Reports", "Inbox", "readme.txt"]) + + browse_folder = self.client.get("/api/v1/files/connectors/profiles/seafile-dev/browse?path=inbox", headers=headers) + self.assertEqual(browse_folder.status_code, 200, browse_folder.text) + self.assertEqual(browse_folder.json()["path"], "inbox") + self.assertEqual(browse_folder.json()["items"][0]["path"], "inbox/imported.txt") + + browse_denied = self.client.get("/api/v1/files/connectors/profiles/seafile-dev/browse?path=blocked", headers=headers) + self.assertEqual(browse_denied.status_code, 403, browse_denied.text) + self.assertIn("connector_policy_denylist", browse_denied.json()["detail"]["requirements"]) + + linked_space = self.client.post( + "/api/v1/files/connector-spaces", + headers=headers, + json={ + "owner_type": "user", + "owner_id": user_id, + "label": "Reports connector", + "connector_profile_id": "seafile-dev", + "library_id": "lib-reports", + "remote_path": "inbox", + }, + ) + self.assertEqual(linked_space.status_code, 201, linked_space.text) + linked_space_payload = linked_space.json() + self.assertEqual(linked_space_payload["label"], "Reports connector") + self.assertEqual(linked_space_payload["owner_type"], "user") + self.assertEqual(linked_space_payload["owner_id"], user_id) + self.assertEqual(linked_space_payload["connector_profile_id"], "seafile-dev") + self.assertEqual(linked_space_payload["provider"], "seafile") + self.assertEqual(linked_space_payload["library_id"], "lib-reports") + self.assertEqual(linked_space_payload["remote_path"], "inbox") + self.assertEqual(linked_space_payload["sync_mode"], "manual") + self.assertTrue(linked_space_payload["read_only"]) + + linked_space_list = self.client.get("/api/v1/files/connector-spaces", headers=headers) + self.assertEqual(linked_space_list.status_code, 200, linked_space_list.text) + self.assertEqual([item["id"] for item in linked_space_list.json()["spaces"]], [linked_space_payload["id"]]) + + all_spaces = self.client.get("/api/v1/files/spaces", headers=headers) + self.assertEqual(all_spaces.status_code, 200, all_spaces.text) + connector_space_entry = next(item for item in all_spaces.json()["spaces"] if item.get("connector_space_id") == linked_space_payload["id"]) + self.assertEqual(connector_space_entry["id"], f"connector:{linked_space_payload['id']}") + self.assertEqual(connector_space_entry["space_type"], "connector") + self.assertEqual(connector_space_entry["connector_profile_id"], "seafile-dev") + self.assertEqual(connector_space_entry["remote_path"], "inbox") + + blocked_space = self.client.post( + "/api/v1/files/connector-spaces", + headers=headers, + json={ + "owner_type": "user", + "owner_id": user_id, + "label": "Blocked connector", + "connector_profile_id": "seafile-dev", + "library_id": "lib-reports", + "remote_path": "blocked", + }, + ) + self.assertEqual(blocked_space.status_code, 403, blocked_space.text) + self.assertIn("connector_policy_denylist", blocked_space.json()["detail"]["requirements"]) + + updated_space = self.client.patch( + f"/api/v1/files/connector-spaces/{linked_space_payload['id']}", + headers=headers, + json={"label": "Reports inbox"}, + ) + self.assertEqual(updated_space.status_code, 200, updated_space.text) + self.assertEqual(updated_space.json()["label"], "Reports inbox") + + blocked_update = self.client.patch( + f"/api/v1/files/connector-spaces/{linked_space_payload['id']}", + headers=headers, + json={"remote_path": "blocked"}, + ) + self.assertEqual(blocked_update.status_code, 403, blocked_update.text) + self.assertIn("connector_policy_denylist", blocked_update.json()["detail"]["requirements"]) + + deleted_space = self.client.delete(f"/api/v1/files/connector-spaces/{linked_space_payload['id']}", headers=headers) + self.assertEqual(deleted_space.status_code, 200, deleted_space.text) + self.assertFalse(deleted_space.json()["is_active"]) + self.assertIsNotNone(deleted_space.json()["deleted_at"]) + + import httpx + + def seafile_request(method: str, url: str, **kwargs): + request = httpx.Request(method, url) + params = kwargs.get("params") or {} + if url == "http://127.0.0.1:9082/api2/auth-token/": + return httpx.Response(200, json={"token": "token-1"}, request=request) + if url == "http://127.0.0.1:9082/api2/repos/repo-1/file/detail/": + self.assertEqual(params.get("p"), "/reports/summary.txt") + self.assertEqual(kwargs.get("headers", {}).get("Authorization"), "Token token-1") + return httpx.Response( + 200, + json={ + "id": "file-1", + "name": "summary.txt", + "size": 14, + "type": "file", + "mtime": 1783425600, + "permission": "r", + }, + request=request, + ) + if url == "http://127.0.0.1:9082/api2/repos/repo-1/file/": + self.assertEqual(params, {"p": "/reports/summary.txt", "reuse": "1"}) + return httpx.Response(200, json="https://download.example.invalid/summary.txt", request=request) + if url == "https://download.example.invalid/summary.txt": + return httpx.Response(200, content=b"summary report", headers={"content-type": "text/plain"}, request=request) + return httpx.Response(404, request=request) + + with patch("govoplan_files.backend.storage.connector_browse.httpx.request", side_effect=seafile_request), patch("govoplan_files.backend.storage.connector_imports.httpx.request", side_effect=seafile_request): + imported = self.client.post( + "/api/v1/files/connectors/profiles/seafile-dev/import", + headers=headers, + json={ + "library_id": "repo-1", + "path": "reports/summary.txt", + "owner_type": "user", + "owner_id": user_id, + "target_folder": "imports", + "metadata": {"case_type": "application"}, + }, + ) + self.assertEqual(imported.status_code, 200, imported.text) + imported_file = imported.json()["files"][0] + self.assertEqual(imported_file["display_path"], "imports/summary.txt") + self.assertEqual(imported_file["source_revision"], "file-1") + self.assertEqual(imported_file["source_provenance"]["connector_id"], "seafile-dev") + self.assertEqual(imported_file["source_provenance"]["provider"], "seafile") + self.assertEqual(imported_file["source_provenance"]["external_id"], "repo-1:reports/summary.txt") + self.assertEqual(imported_file["source_provenance"]["external_path"], "reports/summary.txt") + self.assertEqual(imported_file["source_provenance"]["metadata"]["library_id"], "repo-1") + self.assertEqual(imported_file["source_provenance"]["metadata"]["case_type"], "application") + + webdav_payload = {"content": b"nextcloud notice", "etag": "\"nextcloud-etag-2\""} + + def webdav_request(method: str, url: str, **kwargs): + request = httpx.Request(method, url) + self.assertEqual(method, "GET") + self.assertEqual(url, "http://127.0.0.1:9081/Shared/notice.txt") + self.assertEqual(kwargs.get("headers", {}).get("Authorization"), "Bearer super-secret-nextcloud") + return httpx.Response( + 200, + content=webdav_payload["content"], + headers={"content-type": "text/plain", "etag": webdav_payload["etag"], "content-length": str(len(webdav_payload["content"]))}, + request=request, + ) + + with patch("govoplan_files.backend.storage.connector_imports.httpx.request", side_effect=webdav_request): + nextcloud_import = self.client.post( + "/api/v1/files/connectors/profiles/user-nextcloud/import", + headers=headers, + json={ + "library_id": "", + "path": "Shared/notice.txt", + "owner_type": "user", + "owner_id": user_id, + "target_folder": "imports", + }, + ) + self.assertEqual(nextcloud_import.status_code, 200, nextcloud_import.text) + nextcloud_file = nextcloud_import.json()["files"][0] + self.assertEqual(nextcloud_file["display_path"], "imports/notice.txt") + self.assertEqual(nextcloud_file["source_revision"], "\"nextcloud-etag-2\"") + self.assertEqual(nextcloud_file["source_provenance"]["provider"], "nextcloud") + self.assertEqual(nextcloud_file["source_provenance"]["external_path"], "Shared/notice.txt") + self.assertEqual(nextcloud_file["source_provenance"]["external_id"], "Shared/notice.txt") + + webdav_payload["content"] = b"nextcloud notice updated" + webdav_payload["etag"] = "\"nextcloud-etag-3\"" + with patch("govoplan_files.backend.storage.connector_imports.httpx.request", side_effect=webdav_request): + nextcloud_sync = self.client.post( + "/api/v1/files/connectors/profiles/user-nextcloud/sync", + headers=headers, + json={ + "library_id": "", + "path": "Shared/notice.txt", + "owner_type": "user", + "owner_id": user_id, + "target_folder": "imports", + "conflict_strategy": "rename", + }, + ) + self.assertEqual(nextcloud_sync.status_code, 200, nextcloud_sync.text) + synced = nextcloud_sync.json() + self.assertEqual(synced["action"], "updated") + self.assertEqual(synced["file"]["id"], nextcloud_file["id"]) + self.assertEqual(synced["previous_version_id"], nextcloud_file["version_id"]) + self.assertNotEqual(synced["current_version_id"], nextcloud_file["version_id"]) + self.assertEqual(synced["file"]["source_revision"], "\"nextcloud-etag-3\"") + self.assertEqual(synced["file"]["size_bytes"], len(webdav_payload["content"])) + + with patch("govoplan_files.backend.storage.connector_imports.httpx.request", side_effect=webdav_request): + nextcloud_sync_unchanged = self.client.post( + "/api/v1/files/connectors/profiles/user-nextcloud/sync", + headers=headers, + json={ + "library_id": "", + "path": "Shared/notice.txt", + "owner_type": "user", + "owner_id": user_id, + "target_folder": "imports", + "conflict_strategy": "rename", + }, + ) + self.assertEqual(nextcloud_sync_unchanged.status_code, 200, nextcloud_sync_unchanged.text) + unchanged = nextcloud_sync_unchanged.json() + self.assertEqual(unchanged["action"], "unchanged") + self.assertEqual(unchanged["file"]["id"], nextcloud_file["id"]) + self.assertEqual(unchanged["current_version_id"], synced["current_version_id"]) + + case = self + + class _FakeSmbStat: + st_size = 18 + st_mtime = 1783425600 + st_mtime_ns = 1783425600000000000 + + class _FakeSmbEntry: + def __init__(self, name: str, is_dir: bool, size: int = 18) -> None: + self.name = name + self._is_dir = is_dir + self._stat = _FakeSmbStat() + self._stat.st_size = size + + def is_dir(self) -> bool: + return self._is_dir + + def stat(self): + return self._stat + + class _FakeSmbScandir: + def __enter__(self): + return iter([ + _FakeSmbEntry("Archive", True, 0), + _FakeSmbEntry("smb-notice.txt", False, 18), + ]) + + def __exit__(self, exc_type, exc, traceback): + return False + + class _FakeSmbFile: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def read(self, size: int) -> bytes: + case.assertGreater(size, 18) + return b"smb connector data" + + class _FakeSmbClient: + def scandir(self, path: str, **kwargs): + case.assertEqual(path, r"\\127.0.0.1\files\root\shared") + case.assertEqual(kwargs["port"], 1445) + case.assertEqual(kwargs["username"], "govoplan") + case.assertEqual(kwargs["password"], "super-secret-smb") + case.assertTrue(kwargs["require_signing"]) + return _FakeSmbScandir() + + def stat(self, path: str, **kwargs): + case.assertEqual(path, r"\\127.0.0.1\files\root\shared\smb-notice.txt") + case.assertEqual(kwargs["port"], 1445) + return _FakeSmbStat() + + def open_file(self, path: str, mode: str, **kwargs): + case.assertEqual(path, r"\\127.0.0.1\files\root\shared\smb-notice.txt") + case.assertEqual(mode, "rb") + case.assertEqual(kwargs["password"], "super-secret-smb") + return _FakeSmbFile() + + fake_smb = _FakeSmbClient() + with patch("govoplan_files.backend.storage.connector_browse._smbclient_module", return_value=fake_smb): + smb_browse = self.client.get("/api/v1/files/connectors/profiles/tenant-smb/browse?path=shared", headers=headers) + self.assertEqual(smb_browse.status_code, 200, smb_browse.text) + self.assertEqual([(item["kind"], item["path"]) for item in smb_browse.json()["items"]], [("folder", "shared/Archive"), ("file", "shared/smb-notice.txt")]) + + with patch("govoplan_files.backend.storage.connector_imports._smbclient_module", return_value=fake_smb): + smb_import = self.client.post( + "/api/v1/files/connectors/profiles/tenant-smb/import", + headers=headers, + json={ + "library_id": "", + "path": "shared/smb-notice.txt", + "owner_type": "user", + "owner_id": user_id, + "target_folder": "imports", + }, + ) + self.assertEqual(smb_import.status_code, 200, smb_import.text) + smb_file = smb_import.json()["files"][0] + self.assertEqual(smb_file["display_path"], "imports/smb-notice.txt") + self.assertEqual(smb_file["source_provenance"]["provider"], "smb") + self.assertEqual(smb_file["source_provenance"]["external_id"], "files:shared/smb-notice.txt") + self.assertEqual(smb_file["source_provenance"]["metadata"]["share"], "files") + + filtered = self.client.get("/api/v1/files/connectors/profiles?provider=smb", headers=headers) + self.assertEqual(filtered.status_code, 200, filtered.text) + self.assertEqual([item["id"] for item in filtered.json()["profiles"]], ["tenant-smb"]) + + hidden = self.client.get("/api/v1/files/connectors/profiles/other-tenant-smb", headers=headers) + self.assertEqual(hidden.status_code, 404, hidden.text) + hidden_browse = self.client.get("/api/v1/files/connectors/profiles/other-tenant-smb/browse", headers=headers) + self.assertEqual(hidden_browse.status_code, 404, hidden_browse.text) + + disabled = self.client.get("/api/v1/files/connectors/profiles?include_disabled=true", headers=headers) + self.assertEqual(disabled.status_code, 200, disabled.text) + self.assertIn("disabled-seafile", {item["id"] for item in disabled.json()["profiles"]}) + + from govoplan_files.backend.storage.connector_browse import ( + parse_webdav_multistatus, + seafile_directory_items_from_payload, + seafile_libraries_from_payload, + ) + + webdav_items = parse_webdav_multistatus( + root_url="http://example.test/remote.php/dav/files/admin/", + current_path="reports", + payload=""" + + + /remote.php/dav/files/admin/reports/ + reports + + + /remote.php/dav/files/admin/reports/summary.pdf + summary.pdf42application/pdf"abc" + +""", + ) + self.assertEqual([(item.kind, item.path, item.size_bytes) for item in webdav_items], [("file", "reports/summary.pdf", 42)]) + seafile_libraries = seafile_libraries_from_payload([ + { + "id": "repo-1", + "name": "Reports", + "size": 1024, + "mtime": 1783425600, + "permission": "r", + "encrypted": False, + "type": "repo", + } + ]) + self.assertEqual(seafile_libraries[0].to_response()["kind"], "library") + self.assertEqual(seafile_libraries[0].to_response()["path"], "repo-1") + self.assertEqual(seafile_libraries[0].to_response()["metadata"]["permission"], "r") + seafile_entries = seafile_directory_items_from_payload( + [ + {"type": "dir", "name": "Inbox", "id": "dir-1", "mtime": 1783425600}, + {"type": "file", "name": "summary.pdf", "id": "file-1", "size": 42, "mtime": 1783425601}, + ], + path="reports", + ) + self.assertEqual([(item.kind, item.path, item.size_bytes) for item in seafile_entries], [("folder", "reports/Inbox", None), ("file", "reports/summary.pdf", 42)]) + finally: + for key, value in previous_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + def test_access_configuration_provider_round_trips_roles_groups_and_assignments(self) -> None: + headers, login = self._login() + tenant_id = str(login["tenant"]["id"]) + + from govoplan_access.backend.configuration_provider import SqlAccessConfigurationProvider + from govoplan_access.backend.db.models import Group, GroupRoleAssignment, Role + from govoplan_core.core.configuration_packages import ( + CONFIGURATION_PROVIDER_CAPABILITY, + ConfigurationExportSelection, + ConfigurationPreflightContext, + apply_configuration_package, + dry_run_configuration_package, + export_configuration_package, + ) + + provider = SqlAccessConfigurationProvider() + package = { + "package_id": "govoplan.access.test", + "name": "Access test package", + "version": "1.0.0", + "required_modules": [{"module_id": "access"}], + "required_capabilities": [CONFIGURATION_PROVIDER_CAPABILITY, "access.configuration"], + "fragments": [ + { + "module_id": "access", + "fragment_type": "roles", + "payload": { + "items": [ + { + "slug": "application-clerk", + "name": "Application clerk", + "description": "Handles applications.", + "permissions": ["admin:users:read"], + } + ] + }, + }, + { + "module_id": "access", + "fragment_type": "groups", + "payload": {"items": [{"slug": "front-office", "name": "Front office"}]}, + }, + { + "module_id": "access", + "fragment_type": "group_role_assignments", + "payload": {"items": [{"group": "front-office", "role": "application-clerk"}]}, + }, + ], + } + context = ConfigurationPreflightContext( + tenant_id=tenant_id, + installed_modules={"access": "0.1.6"}, + capabilities=frozenset({CONFIGURATION_PROVIDER_CAPABILITY, "access.configuration"}), + ) + + dry_run = dry_run_configuration_package(package, [provider], context) + self.assertFalse([item for item in dry_run.diagnostics if item.severity == "blocker"], [item.to_dict() for item in dry_run.diagnostics]) + self.assertEqual([item.action for item in dry_run.plan], ["create", "create", "bind"]) + + applied = apply_configuration_package(package, [provider], context) + self.assertFalse([item for item in applied.diagnostics if item.severity == "blocker"], [item.to_dict() for item in applied.diagnostics]) + self.assertIn("application-clerk", applied.created_refs) + self.assertIn("front-office", applied.created_refs) + self.assertIn("front-office:application-clerk", applied.created_refs) + + with SessionLocal() as session: + role = session.query(Role).filter(Role.tenant_id == tenant_id, Role.slug == "application-clerk").one() + group = session.query(Group).filter(Group.tenant_id == tenant_id, Group.slug == "front-office").one() + assignment = session.query(GroupRoleAssignment).filter(GroupRoleAssignment.tenant_id == tenant_id, GroupRoleAssignment.group_id == group.id, GroupRoleAssignment.role_id == role.id).one_or_none() + self.assertIsNotNone(assignment) + self.assertEqual(role.permissions, ["admin:users:read"]) + + repeat = apply_configuration_package(package, [provider], context) + self.assertIn("application-clerk", repeat.updated_refs) + self.assertNotIn("front-office:application-clerk", repeat.created_refs) + + exported = export_configuration_package([provider], ConfigurationExportSelection(tenant_id=tenant_id, module_ids=("access",)), context) + fragments = {fragment.fragment_type: fragment for fragment in exported.fragments} + self.assertIn("application-clerk", {item["slug"] for item in fragments["roles"].payload["items"]}) + self.assertIn("front-office", {item["slug"] for item in fragments["groups"].payload["items"]}) + self.assertIn({"group": "front-office", "role": "application-clerk"}, fragments["group_role_assignments"].payload["items"]) + + endpoint_package = { + **package, + "package_id": "govoplan.access.endpoint-test", + "fragments": [ + { + "module_id": "access", + "fragment_type": "roles", + "payload": {"items": [{"slug": "application-reviewer", "name": "Application reviewer", "permissions": ["admin:users:read"]}]}, + }, + { + "module_id": "access", + "fragment_type": "groups", + "payload": {"items": [{"slug": "review-board", "name": "Review board"}]}, + }, + { + "module_id": "access", + "fragment_type": "group_role_assignments", + "payload": {"items": [{"group": "review-board", "role": "application-reviewer"}]}, + }, + ], + } + catalog = self.client.get("/api/v1/admin/configuration-packages/catalog", headers=headers) + self.assertEqual(catalog.status_code, 200, catalog.text) + self.assertIn("valid", catalog.json()["validation"]) + + endpoint_dry_run = self.client.post( + "/api/v1/admin/configuration-packages/dry-run", + headers=headers, + json={"tenant_id": tenant_id, "package": endpoint_package}, + ) + self.assertEqual(endpoint_dry_run.status_code, 200, endpoint_dry_run.text) + self.assertEqual([item["action"] for item in endpoint_dry_run.json()["plan"]], ["create", "create", "bind"]) + + settings_response = self.client.get("/api/v1/admin/system/settings", headers=headers) + self.assertEqual(settings_response.status_code, 200, settings_response.text) + maintenance_enabled = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": settings_response.json()["default_locale"], + "allow_tenant_custom_groups": settings_response.json()["allow_tenant_custom_groups"], + "allow_tenant_custom_roles": settings_response.json()["allow_tenant_custom_roles"], + "allow_tenant_api_keys": settings_response.json()["allow_tenant_api_keys"], + "maintenance_mode": {"enabled": True, "message": "Configuration package apply test"}, + }, + ) + self.assertEqual(maintenance_enabled.status_code, 200, maintenance_enabled.text) + approver_headers = self._create_system_approver(headers, tenant_id=tenant_id) + change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="configuration_packages.apply", + value=endpoint_package, + target={"tenant_id": tenant_id}, + ) + endpoint_apply = self.client.post( + "/api/v1/admin/configuration-packages/apply", + headers=headers, + json={"tenant_id": tenant_id, "package": endpoint_package, "change_request_id": change_request_id}, + ) + self.assertEqual(endpoint_apply.status_code, 200, endpoint_apply.text) + self.assertIn("application-reviewer", endpoint_apply.json()["created_refs"]) + changes = self.client.get("/api/v1/admin/configuration-changes", headers=headers) + self.assertEqual(changes.status_code, 200, changes.text) + self.assertIn("configuration_packages.apply", {item["key"] for item in changes.json()["history"]}) + applied_request = next(item for item in changes.json()["requests"] if item["id"] == change_request_id) + self.assertEqual(applied_request["status"], "applied") + + endpoint_export = self.client.post( + "/api/v1/admin/configuration-packages/export", + headers=headers, + json={"tenant_id": tenant_id, "module_ids": ["access"]}, + ) + self.assertEqual(endpoint_export.status_code, 200, endpoint_export.text) + exported_roles = next(fragment for fragment in endpoint_export.json()["fragments"] if fragment["fragment_type"] == "roles") + self.assertIn("application-reviewer", {item["slug"] for item in exported_roles["payload"]["items"]}) + + def test_configuration_safety_admin_endpoints_explain_guardrails(self) -> None: + headers, _login = self._login() + catalog = self.client.get("/api/v1/admin/configuration-safety", headers=headers) + self.assertEqual(catalog.status_code, 200, catalog.text) + field_keys = {item["key"] for item in catalog.json()["fields"]} + self.assertIn("files.connector_profiles", field_keys) + self.assertNotIn("DATABASE_URL", field_keys) + + blocked = self.client.post( + "/api/v1/admin/configuration-safety/plan", + headers=headers, + json={ + "key": "files.connector_profiles", + "value": {"password": "plain-secret"}, + "dry_run": False, + "approval_count": 1, + }, + ) + self.assertEqual(blocked.status_code, 200, blocked.text) + plan = blocked.json()["plan"] + self.assertFalse(plan["allowed"]) + self.assertIn("dry_run_required", plan["blockers"]) + self.assertIn("secret_reference_required", plan["blockers"]) + self.assertEqual(plan["audit_event"], "files.connector_profile.updated") + + def test_module_state_update_uses_change_request_and_maintenance_mode(self) -> None: + headers, _login = self._login() + catalog = self.client.get("/api/v1/admin/system/modules", headers=headers) + self.assertEqual(catalog.status_code, 200, catalog.text) + enabled_modules = catalog.json()["desired_enabled"] + + blocked = self.client.put( + "/api/v1/admin/system/modules", + headers=headers, + json={"enabled_modules": enabled_modules}, + ) + self.assertEqual(blocked.status_code, 409, blocked.text) + blocked_detail = blocked.json()["detail"] + self.assertEqual(blocked_detail["code"], "configuration_request_required") + self.assertIn("dry_run_required", blocked_detail["plan"]["blockers"]) + self.assertIn("maintenance_mode_required", blocked_detail["plan"]["blockers"]) + + created = self.client.post( + "/api/v1/admin/configuration-change-requests", + headers=headers, + json={ + "key": "module_management.desired_enabled", + "value": {"enabled_modules": enabled_modules}, + "dry_run": True, + "target": {"scope": "system"}, + "reason": "module-state smoke test", + }, + ) + self.assertEqual(created.status_code, 201, created.text) + request = created.json()["request"] + self.assertEqual(request["status"], "approved") + self.assertTrue(request["plan"]["dry_run_satisfied"]) + + settings_response = self.client.get("/api/v1/admin/system/settings", headers=headers) + self.assertEqual(settings_response.status_code, 200, settings_response.text) + settings_payload = settings_response.json() + maintenance_enabled = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": settings_payload["default_locale"], + "allow_tenant_custom_groups": settings_payload["allow_tenant_custom_groups"], + "allow_tenant_custom_roles": settings_payload["allow_tenant_custom_roles"], + "allow_tenant_api_keys": settings_payload["allow_tenant_api_keys"], + "maintenance_mode": {"enabled": True, "message": "Module state update test"}, + }, + ) + self.assertEqual(maintenance_enabled.status_code, 200, maintenance_enabled.text) + + applied = self.client.put( + "/api/v1/admin/system/modules", + headers=headers, + json={"enabled_modules": enabled_modules, "change_request_id": request["id"]}, + ) + self.assertEqual(applied.status_code, 200, applied.text) + self.assertEqual(set(applied.json()["desired_enabled"]), set(enabled_modules)) + + changes = self.client.get("/api/v1/admin/configuration-changes", headers=headers) + self.assertEqual(changes.status_code, 200, changes.text) + applied_request = next(item for item in changes.json()["requests"] if item["id"] == request["id"]) + self.assertEqual(applied_request["status"], "applied") + self.assertIn("module_management.desired_enabled", {item["key"] for item in changes.json()["history"]}) + + def test_zip_upload_spools_archive_instead_of_full_buffering(self) -> None: + headers, login = self._login() + user_id = str(login["user"]["id"]) + payload = io.BytesIO() + with zipfile.ZipFile(payload, mode="w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("reports/january.txt", "January report") + archive.writestr("reports/february.txt", "February report") + payload.seek(0) + + with patch("govoplan_files.backend.router._read_limited_upload", side_effect=AssertionError("ZIP upload should not be fully buffered")): + response = self.client.post( + "/api/v1/files/upload-zip", + headers=headers, + data={"owner_type": "user", "owner_id": user_id, "path": "imports"}, + files={"file": ("reports.zip", payload.getvalue(), "application/zip")}, + ) + + self.assertEqual(response.status_code, 200, response.text) + paths = sorted(item["display_path"] for item in response.json()["files"]) + self.assertEqual(paths, ["imports/reports/february.txt", "imports/reports/january.txt"]) + def test_group_read_admin_does_not_grant_file_space_admin(self) -> None: owner_headers, _ = self._login() group = self.client.post( @@ -815,6 +2689,210 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(group_listing.status_code, 403, group_listing.text) self.assertEqual(self.client.get(f"/api/v1/files/{file_id}", headers=reader_headers).status_code, 404) + def test_campaign_delta_tracks_create_update_and_delete_tombstone(self) -> None: + headers, _ = self._login() + + full = self.client.get("/api/v1/campaigns/delta", headers=headers) + self.assertEqual(full.status_code, 200, full.text) + self.assertTrue(full.json()["full"]) + watermark = full.json()["watermark"] + + created = self.client.post( + "/api/v1/campaigns/new", + headers=headers, + json={"external_id": "delta-campaign", "name": "Delta campaign"}, + ) + self.assertEqual(created.status_code, 200, created.text) + campaign_id = created.json()["campaign"]["id"] + + after_create = self.client.get( + "/api/v1/campaigns/delta", + headers=headers, + params={"since": watermark}, + ) + self.assertEqual(after_create.status_code, 200, after_create.text) + create_payload = after_create.json() + self.assertFalse(create_payload["full"]) + self.assertIn(campaign_id, {item["id"] for item in create_payload["campaigns"]}) + + updated = self.client.put( + f"/api/v1/campaigns/{campaign_id}", + headers=headers, + json={"name": "Delta campaign updated"}, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + after_update = self.client.get( + "/api/v1/campaigns/delta", + headers=headers, + params={"since": create_payload["watermark"]}, + ) + self.assertEqual(after_update.status_code, 200, after_update.text) + updated_campaigns = {item["id"]: item for item in after_update.json()["campaigns"]} + self.assertEqual(updated_campaigns[campaign_id]["name"], "Delta campaign updated") + + deleted = self.client.delete(f"/api/v1/campaigns/{campaign_id}", headers=headers) + self.assertEqual(deleted.status_code, 204, deleted.text) + + after_delete = self.client.get( + "/api/v1/campaigns/delta", + headers=headers, + params={"since": after_update.json()["watermark"]}, + ) + self.assertEqual(after_delete.status_code, 200, after_delete.text) + deleted_items = after_delete.json()["deleted"] + self.assertEqual(len(deleted_items), 1) + self.assertEqual(deleted_items[0]["id"], campaign_id) + self.assertEqual(deleted_items[0]["resource_type"], "campaign") + self.assertTrue(str(deleted_items[0]["revision"]).startswith("seq:")) + + def test_campaign_workspace_delta_tracks_version_autosave(self) -> None: + headers, _ = self._login() + created = self.client.post( + "/api/v1/campaigns/new", + headers=headers, + json={"external_id": "workspace-delta", "name": "Workspace delta"}, + ) + self.assertEqual(created.status_code, 200, created.text) + campaign_id = created.json()["campaign"]["id"] + version_id = created.json()["version"]["id"] + + full = self.client.get( + f"/api/v1/campaigns/{campaign_id}/workspace/delta", + headers=headers, + params={"include_summary": False}, + ) + self.assertEqual(full.status_code, 200, full.text) + full_payload = full.json() + self.assertTrue(full_payload["full"]) + self.assertEqual(full_payload["current_version"]["id"], version_id) + + autosaved = self.client.post( + f"/api/v1/campaigns/{campaign_id}/versions/{version_id}/autosave", + headers=headers, + json={"current_step": "fields"}, + ) + self.assertEqual(autosaved.status_code, 200, autosaved.text) + self.assertEqual(autosaved.json()["current_step"], "fields") + + delta = self.client.get( + f"/api/v1/campaigns/{campaign_id}/workspace/delta", + headers=headers, + params={"since": full_payload["watermark"], "include_summary": False}, + ) + self.assertEqual(delta.status_code, 200, delta.text) + delta_payload = delta.json() + self.assertFalse(delta_payload["full"]) + self.assertEqual(delta_payload["current_version"]["id"], version_id) + self.assertEqual(delta_payload["current_version"]["current_step"], "fields") + self.assertIn(version_id, {item["id"] for item in delta_payload["versions"]}) + + def test_campaign_jobs_delta_tracks_single_job_update(self) -> None: + headers, _ = self._login() + campaign_id, version_id = self._create_built_delivery_campaign( + headers, + external_id="jobs-delta", + recipient_count=2, + ) + + full = self.client.get( + f"/api/v1/campaigns/{campaign_id}/jobs/delta", + headers=headers, + params={"version_id": version_id, "page_size": 50}, + ) + self.assertEqual(full.status_code, 200, full.text) + full_payload = full.json() + self.assertTrue(full_payload["full"]) + self.assertEqual(len(full_payload["jobs"]), 2) + job_id = full_payload["jobs"][0]["id"] + + from govoplan_campaign.backend.db.models import CampaignJob + + with SessionLocal() as session: + job = session.get(CampaignJob, job_id) + self.assertIsNotNone(job) + job.send_status = "outcome_unknown" + job.last_error = "SMTP outcome needs reconciliation" + session.add(job) + session.commit() + + delta = self.client.get( + f"/api/v1/campaigns/{campaign_id}/jobs/delta", + headers=headers, + params={"version_id": version_id, "since": full_payload["watermark"], "page_size": 50}, + ) + self.assertEqual(delta.status_code, 200, delta.text) + delta_payload = delta.json() + self.assertFalse(delta_payload["full"]) + self.assertEqual([item["id"] for item in delta_payload["jobs"]], [job_id]) + self.assertEqual(delta_payload["jobs"][0]["send_status"], "outcome_unknown") + self.assertEqual(delta_payload["jobs"][0]["last_error"], "SMTP outcome needs reconciliation") + self.assertEqual(delta_payload["counts"]["send"]["outcome_unknown"], 1) + self.assertTrue(str(delta_payload["watermark"]).startswith("seq:")) + + def test_campaign_jobs_cursor_pagination_anchors_delta_pages(self) -> None: + headers, _ = self._login() + campaign_id, version_id = self._create_built_delivery_campaign( + headers, + external_id="jobs-cursor-delta", + recipient_count=5, + ) + + first_page = self.client.get( + f"/api/v1/campaigns/{campaign_id}/jobs/delta", + headers=headers, + params={"version_id": version_id, "page": 1, "page_size": 2}, + ) + self.assertEqual(first_page.status_code, 200, first_page.text) + first_payload = first_page.json() + self.assertTrue(first_payload["full"]) + self.assertTrue(first_payload["next_cursor"]) + + second_page = self.client.get( + f"/api/v1/campaigns/{campaign_id}/jobs/delta", + headers=headers, + params={"version_id": version_id, "page": 2, "page_size": 2, "cursor": first_payload["next_cursor"]}, + ) + self.assertEqual(second_page.status_code, 200, second_page.text) + second_payload = second_page.json() + self.assertEqual(second_payload["cursor"], first_payload["next_cursor"]) + second_page_ids = [item["id"] for item in second_payload["jobs"]] + self.assertEqual(len(second_page_ids), 2) + + mismatched_cursor = self.client.get( + f"/api/v1/campaigns/{campaign_id}/jobs", + headers=headers, + params={"version_id": version_id, "page": 2, "page_size": 3, "cursor": first_payload["next_cursor"]}, + ) + self.assertEqual(mismatched_cursor.status_code, 400, mismatched_cursor.text) + + from govoplan_campaign.backend.db.models import CampaignJob + + first_page_job_id = first_payload["jobs"][0]["id"] + with SessionLocal() as session: + job = session.get(CampaignJob, first_page_job_id) + self.assertIsNotNone(job) + job.send_status = "outcome_unknown" + job.last_error = "Cursor page delta should stay on page one" + session.add(job) + session.commit() + + cursor_delta = self.client.get( + f"/api/v1/campaigns/{campaign_id}/jobs/delta", + headers=headers, + params={ + "version_id": version_id, + "page": 2, + "page_size": 2, + "cursor": first_payload["next_cursor"], + "since": second_payload["watermark"], + }, + ) + self.assertEqual(cursor_delta.status_code, 200, cursor_delta.text) + cursor_delta_payload = cursor_delta.json() + self.assertFalse(cursor_delta_payload["full"]) + self.assertEqual(cursor_delta_payload["jobs"], []) + def test_temporary_and_permanent_user_lock_lifecycle(self) -> None: headers, _ = self._login() created = self.client.post( @@ -1261,6 +3339,172 @@ class ApiSmokeTests(unittest.TestCase): "202605-010001-90100010-9601741.XLSX", }) + def test_managed_attachment_send_uses_frozen_build_artifact_after_file_changes(self) -> None: + headers, login = self._login() + user_id = login["user"]["id"] + campaign_json = { + "version": "1.0", + "campaign": {"id": "managed-send-freeze", "name": "Managed send freeze", "mode": "test"}, + "fields": [], + "global_values": {}, + "server": { + "smtp": { + "host": "mock.smtp", + "port": 2525, + "username": "sender@example.org", + "password": "test-secret", + "security": "starttls", + } + }, + "recipients": { + "from": {"email": "sender@example.org", "name": "Sender", "type": "to"}, + "allow_individual_to": True, + }, + "template": {"subject": "Frozen evidence", "text": "Please see the evidence."}, + "attachments": { + "base_path": "evidence", + "base_paths": [ + { + "id": "managed-evidence", + "name": "Managed evidence", + "source": f"managed:user:{user_id}", + "path": "evidence", + } + ], + "global": [ + { + "id": "proof", + "label": "Proof", + "base_path_id": "managed-evidence", + "base_dir": "evidence", + "file_filter": "proof.txt", + "required": True, + } + ], + "allow_individual": False, + }, + "entries": { + "inline": [ + { + "id": "recipient-1", + "to": [{"email": "recipient@example.org", "type": "to"}], + "fields": {}, + } + ] + }, + "validation_policy": {"missing_email": "block", "template_error": "block", "missing_required_attachment": "block"}, + "delivery": {"rate_limit": {"messages_per_minute": 60}, "imap_append_sent": {"enabled": False}}, + "status_tracking": {"enabled": True}, + } + created = self.client.post("/api/v1/campaigns", headers=headers, json={"config": campaign_json}) + self.assertEqual(created.status_code, 200, created.text) + campaign_id = created.json()["campaign"]["id"] + version_id = created.json()["version"]["id"] + + uploaded = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={ + "owner_type": "user", + "owner_id": user_id, + "path": "evidence", + "campaign_id": campaign_id, + "source_revision": "nextcloud-etag-1", + "source_provenance_json": json.dumps( + { + "source_type": "connector", + "connector_id": "nextcloud-main", + "provider": "nextcloud", + "external_id": "file-123", + "external_path": "/Shared/proof.txt", + } + ), + }, + files=[("files", ("proof.txt", b"original evidence", "text/plain"))], + ) + self.assertEqual(uploaded.status_code, 200, uploaded.text) + + validated = self.client.post(f"/api/v1/campaigns/versions/{version_id}/validate", headers=headers, json={"check_files": True}) + self.assertEqual(validated.status_code, 200, validated.text) + self.assertTrue(validated.json()["ok"], validated.text) + + built = self.client.post(f"/api/v1/campaigns/versions/{version_id}/build", headers=headers, json={"write_eml": True}) + self.assertEqual(built.status_code, 200, built.text) + self.assertEqual(built.json()["built_count"], 1) + managed_match = built.json()["messages"][0]["attachments"][0]["managed_matches"][0] + self.assertEqual(managed_match["source_revision"], "nextcloud-etag-1") + self.assertEqual(managed_match["source_provenance"]["connector_id"], "nextcloud-main") + self.assertEqual(managed_match["source_provenance"]["revision"], "nextcloud-etag-1") + + from govoplan_campaign.backend.db.models import CampaignJob + from govoplan_files.backend.db.models import CampaignAttachmentUse, FileBlob + from govoplan_files.backend.storage.backends import get_storage_backend + from govoplan_mail.backend.dev.mock_mailbox import list_records + + with SessionLocal() as session: + job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one() + self.assertTrue(job.eml_local_path) + built_use = ( + session.query(CampaignAttachmentUse) + .filter( + CampaignAttachmentUse.campaign_job_id == job.id, + CampaignAttachmentUse.use_stage == "built", + ) + .one() + ) + original_version_id = built_use.file_version_id + original_blob_id = built_use.file_blob_id + original_storage_key = session.get(FileBlob, original_blob_id).storage_key + + get_storage_backend().delete(original_storage_key) + changed = self.client.post( + "/api/v1/files/upload", + headers=headers, + data={ + "owner_type": "user", + "owner_id": user_id, + "path": "evidence", + "campaign_id": campaign_id, + "conflict_strategy": "overwrite", + }, + files=[("files", ("proof.txt", b"changed live content", "text/plain"))], + ) + self.assertEqual(changed.status_code, 200, changed.text) + + sent = self.client.post( + f"/api/v1/campaigns/{campaign_id}/send-now", + headers=headers, + json={ + "version_id": version_id, + "validate_before_send": False, + "build_before_send": False, + "use_rate_limit": False, + "enqueue_imap_task": False, + }, + ) + self.assertEqual(sent.status_code, 200, sent.text) + self.assertEqual(sent.json()["result"]["sent_count"], 1, sent.text) + + records = list_records(kind="smtp") + self.assertEqual(len(records), 1) + raw_filename = records[0].get("raw_filename") + self.assertTrue(raw_filename) + captured = BytesParser(policy=policy.default).parsebytes((_TEST_ROOT / "mock-mailbox" / "messages" / str(raw_filename)).read_bytes()) + attachments = {part.get_filename(): part.get_payload(decode=True) for part in captured.iter_attachments()} + self.assertEqual(attachments["proof.txt"], b"original evidence") + + with SessionLocal() as session: + sent_use = ( + session.query(CampaignAttachmentUse) + .filter( + CampaignAttachmentUse.campaign_id == campaign_id, + CampaignAttachmentUse.use_stage == "sent", + ) + .one() + ) + self.assertEqual(sent_use.file_version_id, original_version_id) + self.assertEqual(sent_use.file_blob_id, original_blob_id) + def test_non_blocking_review_conditions_can_be_accepted_in_bulk(self) -> None: headers, _ = self._login() @@ -2459,6 +4703,623 @@ class ApiSmokeTests(unittest.TestCase): + def test_access_admin_users_delta_tracks_create_and_update(self) -> None: + headers, _ = self._login() + + initial = self.client.get("/api/v1/admin/users/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + created = self.client.post( + "/api/v1/admin/users", + headers=headers, + json={ + "email": "delta-user@example.local", + "display_name": "Delta User", + "password": "delta-user-password", + "password_reset_required": False, + "is_active": True, + "group_ids": [], + "role_ids": [], + }, + ) + self.assertEqual(created.status_code, 201, created.text) + user_id = created.json()["user"]["id"] + + created_delta = self.client.get( + "/api/v1/admin/users/delta", + headers=headers, + params={"since": initial_payload["watermark"]}, + ) + self.assertEqual(created_delta.status_code, 200, created_delta.text) + created_payload = created_delta.json() + self.assertFalse(created_payload["full"]) + self.assertEqual(created_payload["deleted"], []) + self.assertIn("delta-user@example.local", {user["email"] for user in created_payload["users"]}) + + updated = self.client.patch( + f"/api/v1/admin/users/{user_id}", + headers=headers, + json={"display_name": "Delta User Updated", "group_ids": [], "role_ids": []}, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + updated_delta = self.client.get( + "/api/v1/admin/users/delta", + headers=headers, + params={"since": created_payload["watermark"]}, + ) + self.assertEqual(updated_delta.status_code, 200, updated_delta.text) + updated_payload = updated_delta.json() + self.assertFalse(updated_payload["full"]) + changed_user = next(user for user in updated_payload["users"] if user["id"] == user_id) + self.assertEqual(changed_user["display_name"], "Delta User Updated") + + def test_access_admin_api_key_delta_returns_tombstone_when_revoked_hidden(self) -> None: + headers, login = self._login() + + initial = self.client.get("/api/v1/admin/api-keys/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + created = self.client.post( + "/api/v1/admin/api-keys", + headers=headers, + json={ + "name": "Delta API key", + "user_id": login["user"]["id"], + "scopes": ["files:read"], + }, + ) + self.assertEqual(created.status_code, 201, created.text) + key_id = created.json()["id"] + + created_delta = self.client.get( + "/api/v1/admin/api-keys/delta", + headers=headers, + params={"since": initial_payload["watermark"]}, + ) + self.assertEqual(created_delta.status_code, 200, created_delta.text) + created_payload = created_delta.json() + self.assertFalse(created_payload["full"]) + self.assertIn(key_id, {item["id"] for item in created_payload["api_keys"]}) + + revoked = self.client.post(f"/api/v1/admin/api-keys/{key_id}/revoke", headers=headers) + self.assertEqual(revoked.status_code, 200, revoked.text) + + hidden_revoked_delta = self.client.get( + "/api/v1/admin/api-keys/delta", + headers=headers, + params={"since": created_payload["watermark"]}, + ) + self.assertEqual(hidden_revoked_delta.status_code, 200, hidden_revoked_delta.text) + hidden_payload = hidden_revoked_delta.json() + self.assertFalse(hidden_payload["full"]) + self.assertTrue( + any(item["id"] == key_id and item["resource_type"] == "access_api_key" for item in hidden_payload["deleted"]), + hidden_payload["deleted"], + ) + + visible_revoked_delta = self.client.get( + "/api/v1/admin/api-keys/delta", + headers=headers, + params={"since": created_payload["watermark"], "include_revoked": "true"}, + ) + self.assertEqual(visible_revoked_delta.status_code, 200, visible_revoked_delta.text) + visible_key = next(item for item in visible_revoked_delta.json()["api_keys"] if item["id"] == key_id) + self.assertIsNotNone(visible_key["revoked_at"]) + + def test_settings_deltas_track_sections_and_system_language_dependency(self) -> None: + headers, _ = self._login() + + system_initial = self.client.get("/api/v1/admin/system/settings/delta", headers=headers) + self.assertEqual(system_initial.status_code, 200, system_initial.text) + system_initial_payload = system_initial.json() + self.assertTrue(system_initial_payload["full"]) + self.assertTrue(system_initial_payload["watermark"]) + + tenant_initial = self.client.get("/api/v1/admin/tenant/settings/delta", headers=headers) + self.assertEqual(tenant_initial.status_code, 200, tenant_initial.text) + tenant_initial_payload = tenant_initial.json() + self.assertTrue(tenant_initial_payload["full"]) + self.assertTrue(tenant_initial_payload["watermark"]) + + system_item = system_initial_payload["item"] + available_languages = list(system_item["available_languages"]) + if "fr" not in {item["code"] for item in available_languages}: + available_languages.append({"code": "fr", "label": "French", "native_label": "Francais"}) + enabled_codes = list(dict.fromkeys([*system_item["enabled_language_codes"], "fr"])) + updated_system = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": system_item["default_locale"], + "allow_tenant_custom_groups": system_item["allow_tenant_custom_groups"], + "allow_tenant_custom_roles": system_item["allow_tenant_custom_roles"], + "allow_tenant_api_keys": system_item["allow_tenant_api_keys"], + "available_languages": available_languages, + "enabled_language_codes": enabled_codes, + }, + ) + self.assertEqual(updated_system.status_code, 200, updated_system.text) + + system_delta = self.client.get( + "/api/v1/admin/system/settings/delta", + headers=headers, + params={"since": system_initial_payload["watermark"]}, + ) + self.assertEqual(system_delta.status_code, 200, system_delta.text) + system_delta_payload = system_delta.json() + self.assertFalse(system_delta_payload["full"]) + self.assertIn("languages", system_delta_payload["changed_sections"]) + self.assertIn("fr", system_delta_payload["sections"]["languages"]["enabled_language_codes"]) + + tenant_delta = self.client.get( + "/api/v1/admin/tenant/settings/delta", + headers=headers, + params={"since": tenant_initial_payload["watermark"]}, + ) + self.assertEqual(tenant_delta.status_code, 200, tenant_delta.text) + tenant_delta_payload = tenant_delta.json() + self.assertFalse(tenant_delta_payload["full"]) + self.assertIn("languages", tenant_delta_payload["changed_sections"]) + self.assertIn("fr", tenant_delta_payload["sections"]["languages"]["system_enabled_language_codes"]) + + def test_tenant_admin_delta_tracks_create_and_update(self) -> None: + headers, _ = self._login() + + initial = self.client.get("/api/v1/admin/tenants/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + created = self.client.post( + "/api/v1/admin/tenants", + headers=headers, + json={"slug": "tenant-delta", "name": "Tenant Delta", "settings": {}}, + ) + self.assertEqual(created.status_code, 201, created.text) + tenant_id = created.json()["id"] + + created_delta = self.client.get( + "/api/v1/admin/tenants/delta", + headers=headers, + params={"since": initial_payload["watermark"]}, + ) + self.assertEqual(created_delta.status_code, 200, created_delta.text) + created_payload = created_delta.json() + self.assertFalse(created_payload["full"]) + self.assertIn("Tenant Delta", {tenant["name"] for tenant in created_payload["tenants"]}) + self.assertFalse(created_payload["deleted"]) + + updated = self.client.patch( + f"/api/v1/admin/tenants/{tenant_id}", + headers=headers, + json={"name": "Tenant Delta Updated", "is_active": False}, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + updated_delta = self.client.get( + "/api/v1/admin/tenants/delta", + headers=headers, + params={"since": created_payload["watermark"]}, + ) + self.assertEqual(updated_delta.status_code, 200, updated_delta.text) + updated_payload = updated_delta.json() + self.assertFalse(updated_payload["full"]) + updated_item = next(tenant for tenant in updated_payload["tenants"] if tenant["id"] == tenant_id) + self.assertEqual(updated_item["name"], "Tenant Delta Updated") + self.assertFalse(updated_item["is_active"]) + + def test_governance_template_delta_tracks_create_update_and_delete(self) -> None: + headers, login = self._login() + tenant_id = login["tenant"]["id"] + approver_headers = self._create_system_approver(headers, tenant_id=tenant_id, email="governance-delta-approver@example.local") + + initial = self.client.get("/api/v1/admin/system/governance-templates/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + create_payload = { + "kind": "group", + "slug": "governance-delta", + "name": "Governance Delta", + "description": "Delta tracked system group", + "permissions": [], + "is_active": True, + "assignments": [{"tenant_id": tenant_id, "mode": "available"}], + } + create_change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="governance_templates", + value=create_payload, + target={"kind": "group", "slug": "governance-delta"}, + ) + created = self.client.post( + "/api/v1/admin/system/governance-templates", + headers=headers, + json={**create_payload, "change_request_id": create_change_request_id}, + ) + self.assertEqual(created.status_code, 201, created.text) + template_id = created.json()["id"] + + created_delta = self.client.get( + "/api/v1/admin/system/governance-templates/delta", + headers=headers, + params={"since": initial_payload["watermark"]}, + ) + self.assertEqual(created_delta.status_code, 200, created_delta.text) + created_payload = created_delta.json() + self.assertFalse(created_payload["full"]) + self.assertIn(template_id, {item["id"] for item in created_payload["templates"]}) + + update_payload = { + "name": "Governance Delta Updated", + "description": "Delta tracked system group updated", + "permissions": [], + "is_active": False, + "assignments": [{"tenant_id": tenant_id, "mode": "available"}], + } + update_change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="governance_templates", + value=update_payload, + target={"kind": "group", "id": template_id}, + ) + updated = self.client.patch( + f"/api/v1/admin/system/governance-templates/{template_id}", + headers=headers, + json={**update_payload, "change_request_id": update_change_request_id}, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + updated_delta = self.client.get( + "/api/v1/admin/system/governance-templates/delta", + headers=headers, + params={"since": created_payload["watermark"]}, + ) + self.assertEqual(updated_delta.status_code, 200, updated_delta.text) + updated_item = next(item for item in updated_delta.json()["templates"] if item["id"] == template_id) + self.assertEqual(updated_item["name"], "Governance Delta Updated") + self.assertFalse(updated_item["is_active"]) + + delete_value = {"id": template_id, "kind": "group", "delete": True} + delete_change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="governance_templates", + value=delete_value, + target={"kind": "group", "id": template_id}, + ) + deleted = self.client.delete( + f"/api/v1/admin/system/governance-templates/{template_id}", + headers=headers, + params={"change_request_id": delete_change_request_id}, + ) + self.assertEqual(deleted.status_code, 204, deleted.text) + + deleted_delta = self.client.get( + "/api/v1/admin/system/governance-templates/delta", + headers=headers, + params={"since": updated_delta.json()["watermark"]}, + ) + self.assertEqual(deleted_delta.status_code, 200, deleted_delta.text) + self.assertTrue(any(item["id"] == template_id and item["resource_type"] == "governance_template" for item in deleted_delta.json()["deleted"])) + + def test_module_installer_history_supports_cursor_windows(self) -> None: + from govoplan_core.core.module_installer import default_installer_runtime_dir + + headers, _ = self._login() + runtime_dir = default_installer_runtime_dir(os.environ["DATABASE_URL"]) + shutil.rmtree(runtime_dir, ignore_errors=True) + for run_id in ("run-001", "run-002", "run-003"): + run_dir = runtime_dir / "runs" / run_id + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "record.json").write_text( + json.dumps({"run_id": run_id, "status": "applied", "started_at": f"2030-01-0{run_id[-1]}T00:00:00+00:00", "commands": [], "plan": []}), + encoding="utf-8", + ) + requests_dir = runtime_dir / "requests" + requests_dir.mkdir(parents=True, exist_ok=True) + for request_id in ("request-001", "request-002", "request-003"): + (requests_dir / f"{request_id}.json").write_text( + json.dumps({"request_id": request_id, "status": "queued", "created_at": f"2030-01-0{request_id[-1]}T00:00:00+00:00", "options": {}}), + encoding="utf-8", + ) + + first_runs = self.client.get( + "/api/v1/admin/system/modules/install-runs", + headers=headers, + params={"page_size": 2}, + ) + self.assertEqual(first_runs.status_code, 200, first_runs.text) + first_runs_payload = first_runs.json() + self.assertEqual([item["run_id"] for item in first_runs_payload["runs"]], ["run-003", "run-002"]) + self.assertTrue(first_runs_payload["next_cursor"]) + + next_runs = self.client.get( + "/api/v1/admin/system/modules/install-runs", + headers=headers, + params={"page_size": 2, "cursor": first_runs_payload["next_cursor"]}, + ) + self.assertEqual(next_runs.status_code, 200, next_runs.text) + self.assertEqual([item["run_id"] for item in next_runs.json()["runs"]], ["run-001"]) + + shutil.rmtree(runtime_dir / "runs" / "run-002", ignore_errors=True) + stale_runs = self.client.get( + "/api/v1/admin/system/modules/install-runs", + headers=headers, + params={"page_size": 2, "cursor": first_runs_payload["next_cursor"]}, + ) + self.assertEqual(stale_runs.status_code, 200, stale_runs.text) + self.assertTrue(stale_runs.json()["full"]) + self.assertEqual([item["run_id"] for item in stale_runs.json()["runs"]], ["run-003", "run-001"]) + + first_requests = self.client.get( + "/api/v1/admin/system/modules/install-requests", + headers=headers, + params={"page_size": 2}, + ) + self.assertEqual(first_requests.status_code, 200, first_requests.text) + first_requests_payload = first_requests.json() + self.assertEqual([item["request_id"] for item in first_requests_payload["requests"]], ["request-003", "request-002"]) + self.assertTrue(first_requests_payload["next_cursor"]) + + next_requests = self.client.get( + "/api/v1/admin/system/modules/install-requests", + headers=headers, + params={"page_size": 2, "cursor": first_requests_payload["next_cursor"]}, + ) + self.assertEqual(next_requests.status_code, 200, next_requests.text) + self.assertEqual([item["request_id"] for item in next_requests.json()["requests"]], ["request-001"]) + + def test_system_accounts_delta_tolerates_repeated_no_change_reload(self) -> None: + headers, _ = self._login() + initial = self.client.get("/api/v1/admin/system/accounts/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + first_reload = self.client.get( + "/api/v1/admin/system/accounts/delta", + headers=headers, + params={"since": initial_payload["watermark"]}, + ) + self.assertEqual(first_reload.status_code, 200, first_reload.text) + first_payload = first_reload.json() + self.assertFalse(first_payload["full"]) + + second_reload = self.client.get( + "/api/v1/admin/system/accounts/delta", + headers=headers, + params={"since": first_payload["watermark"]}, + ) + self.assertEqual(second_reload.status_code, 200, second_reload.text) + second_payload = second_reload.json() + self.assertFalse(second_payload["full"]) + + def test_configuration_changes_delta_tracks_requests(self) -> None: + headers, _ = self._login() + + initial = self.client.get("/api/v1/admin/configuration-changes/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + created = self.client.post( + "/api/v1/admin/configuration-change-requests", + headers=headers, + json={ + "key": "maintenance_mode", + "value": {"enabled": False, "message": "delta request"}, + "dry_run": True, + "target": {"scope": "system"}, + "reason": "delta smoke test", + }, + ) + self.assertEqual(created.status_code, 201, created.text) + request_id = created.json()["request"]["id"] + + delta = self.client.get( + "/api/v1/admin/configuration-changes/delta", + headers=headers, + params={"since": initial_payload["watermark"]}, + ) + self.assertEqual(delta.status_code, 200, delta.text) + delta_payload = delta.json() + self.assertFalse(delta_payload["full"]) + self.assertIn(request_id, {item["id"] for item in delta_payload["requests"]}) + + def test_admin_audit_delta_tracks_new_events(self) -> None: + headers, _ = self._login() + + initial = self.client.get("/api/v1/admin/audit/delta", headers=headers, params={"scope": "tenant", "page_size": 25}) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + self.assertTrue(initial_payload["watermark"]) + + created = self.client.post( + "/api/v1/admin/users", + headers=headers, + json={ + "email": "audit-delta-user@example.local", + "display_name": "Audit Delta User", + "password": "audit-delta-password", + "password_reset_required": False, + "is_active": True, + "group_ids": [], + "role_ids": [], + }, + ) + self.assertEqual(created.status_code, 201, created.text) + + delta = self.client.get( + "/api/v1/admin/audit/delta", + headers=headers, + params={"scope": "tenant", "page_size": 25, "since": initial_payload["watermark"]}, + ) + self.assertEqual(delta.status_code, 200, delta.text) + delta_payload = delta.json() + self.assertFalse(delta_payload["full"]) + self.assertIn("user.created", {item["action"] for item in delta_payload["items"]}) + + def test_access_admin_users_delta_paginates_enforces_permissions_and_tenant_scope(self) -> None: + headers, _ = self._login() + + initial = self.client.get("/api/v1/admin/users/delta", headers=headers) + self.assertEqual(initial.status_code, 200, initial.text) + initial_watermark = initial.json()["watermark"] + + for index in range(2): + created = self.client.post( + "/api/v1/admin/users", + headers=headers, + json={ + "email": f"delta-page-{index}@example.local", + "display_name": f"Delta Page {index}", + "password": "delta-page-password", + "password_reset_required": False, + "is_active": True, + "group_ids": [], + "role_ids": [], + }, + ) + self.assertEqual(created.status_code, 201, created.text) + + page_one = self.client.get( + "/api/v1/admin/users/delta", + headers=headers, + params={"since": initial_watermark, "limit": 1}, + ) + self.assertEqual(page_one.status_code, 200, page_one.text) + page_one_payload = page_one.json() + self.assertFalse(page_one_payload["full"]) + self.assertTrue(page_one_payload["has_more"]) + self.assertEqual(len(page_one_payload["users"]), 1) + + page_two = self.client.get( + "/api/v1/admin/users/delta", + headers=headers, + params={"since": page_one_payload["watermark"], "limit": 1}, + ) + self.assertEqual(page_two.status_code, 200, page_two.text) + self.assertFalse(page_two.json()["full"]) + self.assertEqual(len(page_two.json()["users"]), 1) + + reader_role = self._create_role(headers, slug="delta-file-reader", permissions=["files:read"]) + self._create_user( + headers, + email="delta-reader@example.local", + password="delta-reader-password", + role_ids=[str(reader_role["id"])], + ) + reader_headers, _ = self._login_as(email="delta-reader@example.local", password="delta-reader-password") + denied = self.client.get("/api/v1/admin/users/delta", headers=reader_headers) + self.assertEqual(denied.status_code, 403, denied.text) + + scoped_initial = self.client.get("/api/v1/admin/users/delta", headers=headers) + self.assertEqual(scoped_initial.status_code, 200, scoped_initial.text) + created_tenant = self.client.post( + "/api/v1/admin/tenants", + headers=headers, + json={"slug": "delta-scope", "name": "Delta scope", "settings": {}}, + ) + self.assertEqual(created_tenant.status_code, 201, created_tenant.text) + switched = self.client.post( + "/api/v1/auth/switch-tenant", + headers=headers, + json={"tenant_id": created_tenant.json()["id"]}, + ) + self.assertEqual(switched.status_code, 200, switched.text) + cross_tenant_user = self.client.post( + "/api/v1/admin/users", + headers=headers, + json={ + "email": "other-tenant-delta@example.local", + "display_name": "Other Tenant Delta", + "password": "other-tenant-delta-password", + "password_reset_required": False, + "is_active": True, + "group_ids": [], + "role_ids": [], + }, + ) + self.assertEqual(cross_tenant_user.status_code, 201, cross_tenant_user.text) + + default_headers, _ = self._login() + scoped_delta = self.client.get( + "/api/v1/admin/users/delta", + headers=default_headers, + params={"since": scoped_initial.json()["watermark"]}, + ) + self.assertEqual(scoped_delta.status_code, 200, scoped_delta.text) + self.assertNotIn("other-tenant-delta@example.local", {user["email"] for user in scoped_delta.json()["users"]}) + + def test_admin_audit_delta_supports_filtered_sorted_first_page(self) -> None: + headers, _ = self._login() + + initial = self.client.get( + "/api/v1/admin/audit/delta", + headers=headers, + params={ + "scope": "system", + "page_size": 25, + "sort_by": "action", + "sort_direction": "asc", + "filter_action": "system_settings", + }, + ) + self.assertEqual(initial.status_code, 200, initial.text) + initial_payload = initial.json() + self.assertTrue(initial_payload["full"]) + + settings_response = self.client.get("/api/v1/admin/system/settings", headers=headers) + self.assertEqual(settings_response.status_code, 200, settings_response.text) + settings_payload = settings_response.json() + updated = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": settings_payload["default_locale"], + "allow_tenant_custom_groups": settings_payload["allow_tenant_custom_groups"], + "allow_tenant_custom_roles": settings_payload["allow_tenant_custom_roles"], + "allow_tenant_api_keys": settings_payload["allow_tenant_api_keys"], + "available_languages": settings_payload["available_languages"], + "enabled_language_codes": settings_payload["enabled_language_codes"], + }, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + delta = self.client.get( + "/api/v1/admin/audit/delta", + headers=headers, + params={ + "scope": "system", + "page_size": 25, + "sort_by": "action", + "sort_direction": "asc", + "filter_action": "system_settings", + "since": initial_payload["watermark"], + }, + ) + self.assertEqual(delta.status_code, 200, delta.text) + delta_payload = delta.json() + self.assertFalse(delta_payload["full"]) + self.assertTrue(all("system_settings" in item["action"] for item in delta_payload["items"])) + self.assertIn("system_settings.updated", {item["action"] for item in delta_payload["items"]}) + def test_system_governance_defaults_templates_and_central_accounts(self) -> None: headers, login = self._login() tenant_id = login["tenant"]["id"] @@ -2467,6 +5328,20 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(settings_response.status_code, 200, settings_response.text) self.assertTrue(settings_response.json()["allow_tenant_custom_groups"]) self.assertEqual(settings_response.json()["privacy_retention_policy"]["audit_detail_level"], "full") + approver_headers = self._create_system_approver(headers, tenant_id=tenant_id) + + privacy_update = { + **settings_response.json()["privacy_retention_policy"], + "audit_detail_level": "redacted", + "mock_mailbox_retention_days": 0, + } + settings_change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="privacy_retention_policy", + value=privacy_update, + target={"scope": "system"}, + ) deny_local_groups = self.client.patch( "/api/v1/admin/system/settings", @@ -2476,10 +5351,8 @@ class ApiSmokeTests(unittest.TestCase): "allow_tenant_custom_groups": False, "allow_tenant_custom_roles": True, "allow_tenant_api_keys": True, - "privacy_retention_policy": { - "audit_detail_level": "redacted", - "mock_mailbox_retention_days": 0, - }, + "privacy_retention_policy": privacy_update, + "change_request_id": settings_change_request_id, }, ) self.assertEqual(deny_local_groups.status_code, 200, deny_local_groups.text) @@ -2493,13 +5366,31 @@ class ApiSmokeTests(unittest.TestCase): ) self.assertEqual(widened_tenant.status_code, 422, widened_tenant.text) + retention_patch = {"allow_lower_level_limits": {"raw_campaign_json_retention_days": False}} + retention_change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="privacy_retention_policy", + value=retention_patch, + target={"scope_type": "system", "scope_id": None}, + ) locked_retention = self.client.put( "/api/v1/admin/privacy-retention/policies/system", headers=headers, - json={"policy": {"allow_lower_level_limits": {"raw_campaign_json_retention_days": False}}}, + json={"policy": retention_patch, "change_request_id": retention_change_request_id}, ) self.assertEqual(locked_retention.status_code, 200, locked_retention.text) self.assertFalse(locked_retention.json()["effective_policy"]["allow_lower_level_limits"]["raw_campaign_json_retention_days"]) + retention_explain = self.client.get( + "/api/v1/admin/privacy-retention/policies/tenant/explain", + headers=headers, + ) + self.assertEqual(retention_explain.status_code, 200, retention_explain.text) + retention_explain_payload = retention_explain.json() + self.assertFalse(retention_explain_payload["decision"]["allowed"]) + self.assertIn("raw_campaign_json_retention_days", retention_explain_payload["blocked_fields"]) + self.assertEqual(retention_explain_payload["parent_policy_sources"][0]["path"], "system") + self.assertEqual(retention_explain_payload["decision"]["source_path"][0]["path"], "system") tenant_retention_widen = self.client.put( "/api/v1/admin/privacy-retention/policies/tenant", headers=headers, @@ -2529,18 +5420,26 @@ class ApiSmokeTests(unittest.TestCase): ) self.assertEqual(blocked_group.status_code, 409, blocked_group.text) + central_group_payload = { + "kind": "group", + "slug": "case-workers", + "name": "Case workers", + "description": "System-controlled group identity", + "permissions": [], + "is_active": True, + "assignments": [{"tenant_id": tenant_id, "mode": "required"}], + } + governance_change_request_id = self._approved_configuration_change( + headers, + approver_headers, + key="governance_templates", + value=central_group_payload, + target={"kind": "group", "slug": "case-workers"}, + ) central_group = self.client.post( "/api/v1/admin/system/governance-templates", headers=headers, - json={ - "kind": "group", - "slug": "case-workers", - "name": "Case workers", - "description": "System-controlled group identity", - "permissions": [], - "is_active": True, - "assignments": [{"tenant_id": tenant_id, "mode": "required"}], - }, + json={**central_group_payload, "change_request_id": governance_change_request_id}, ) self.assertEqual(central_group.status_code, 201, central_group.text) template_id = central_group.json()["id"] @@ -2598,6 +5497,11 @@ class ApiSmokeTests(unittest.TestCase): self.assertIn("system_settings.updated", actions) self.assertIn("governance_template.created", actions) self.assertIn("system_account.created", actions) + changes = self.client.get("/api/v1/admin/configuration-changes", headers=headers) + self.assertEqual(changes.status_code, 200, changes.text) + history_keys = {item["key"] for item in changes.json()["history"]} + self.assertIn("privacy_retention_policy", history_keys) + self.assertIn("governance_templates", history_keys) def test_refined_permissions_are_narrow_and_enforce_delegation_ceiling(self) -> None: @@ -2721,12 +5625,14 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(self.client.get(f"/api/v1/campaigns/{campaign_id}", headers=other_headers).status_code, 403) def test_policy_responses_include_source_provenance(self) -> None: - headers, _ = self._login() + headers, login = self._login() + tenant_id = login["tenant"]["id"] retention = self.client.get("/api/v1/admin/privacy-retention/policies/tenant", headers=headers) self.assertEqual(retention.status_code, 200, retention.text) retention_payload = retention.json() self.assertEqual([item["label"] for item in retention_payload["effective_policy_sources"]], ["System", "Tenant"]) + self.assertEqual([item["path"] for item in retention_payload["effective_policy_sources"]], ["system", f"tenant:{tenant_id}"]) self.assertEqual([item["label"] for item in retention_payload["parent_policy_sources"]], ["System"]) self.assertIn("defaults", retention_payload["effective_policy_sources"][0]["applied_fields"]) @@ -2749,6 +5655,7 @@ class ApiSmokeTests(unittest.TestCase): campaign_source = payload["effective_policy_sources"][-1] self.assertEqual(campaign_source["scope_type"], "campaign") self.assertEqual(campaign_source["scope_id"], campaign_id) + self.assertEqual(campaign_source["path"], f"campaign:{campaign_id}") self.assertIn("allow_campaign_profiles", campaign_source["applied_fields"]) def test_campaign_scoped_mail_profile_policy_is_enforced(self) -> None: @@ -3026,15 +5933,37 @@ class ApiSmokeTests(unittest.TestCase): profile = self.client.patch( "/api/v1/auth/profile", headers=headers, - json={"display_name": "Global Account Name", "tenant_display_name": "Tenant Alias"}, + json={ + "display_name": "Global Account Name", + "tenant_display_name": "Tenant Alias", + "ui_preferences": { + "compact_tables": True, + "show_inline_help_hints": False, + "reduce_motion": True, + "sticky_section_sidebars": False, + "theme": "dark", + }, + }, ) self.assertEqual(profile.status_code, 200, profile.text) self.assertEqual(profile.json()["user"]["display_name"], "Global Account Name") self.assertEqual(profile.json()["user"]["tenant_display_name"], "Tenant Alias") + self.assertEqual( + profile.json()["user"]["ui_preferences"], + { + "compact_tables": True, + "show_inline_help_hints": False, + "reduce_motion": True, + "sticky_section_sidebars": False, + "theme": "dark", + }, + ) refreshed = self.client.get("/api/v1/auth/me", headers=headers) self.assertEqual(refreshed.status_code, 200, refreshed.text) self.assertEqual(refreshed.json()["user"]["display_name"], "Global Account Name") self.assertEqual(refreshed.json()["user"]["tenant_display_name"], "Tenant Alias") + self.assertEqual(refreshed.json()["user"]["ui_preferences"]["theme"], "dark") + self.assertFalse(refreshed.json()["user"]["ui_preferences"]["show_inline_help_hints"]) roles = self.client.get("/api/v1/admin/system/roles", headers=headers) self.assertEqual(roles.status_code, 200, roles.text) @@ -3114,6 +6043,120 @@ class ApiSmokeTests(unittest.TestCase): self.assertEqual(len(second_page.json()["items"]), 2) + def test_admin_audit_cursor_pagination_is_stable_after_newer_insert(self) -> None: + headers, _ = self._login() + for index in range(6): + updated = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": f"en-{index}", + "allow_tenant_custom_groups": True, + "allow_tenant_custom_roles": True, + "allow_tenant_api_keys": True, + }, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + first_page = self.client.get( + "/api/v1/admin/audit", + headers=headers, + params={ + "scope": "system", + "page": 1, + "page_size": 2, + "sort_by": "time", + "sort_direction": "desc", + "filter_action": "system_settings", + }, + ) + self.assertEqual(first_page.status_code, 200, first_page.text) + first_payload = first_page.json() + self.assertTrue(first_payload["next_cursor"]) + + second_page = self.client.get( + "/api/v1/admin/audit", + headers=headers, + params={ + "scope": "system", + "page": 2, + "page_size": 2, + "cursor": first_payload["next_cursor"], + "sort_by": "time", + "sort_direction": "desc", + "filter_action": "system_settings", + }, + ) + self.assertEqual(second_page.status_code, 200, second_page.text) + second_payload = second_page.json() + second_page_ids = [item["id"] for item in second_payload["items"]] + self.assertEqual(second_payload["cursor"], first_payload["next_cursor"]) + self.assertEqual(len(second_page_ids), 2) + + second_page_full_delta = self.client.get( + "/api/v1/admin/audit/delta", + headers=headers, + params={ + "scope": "system", + "page_size": 2, + "cursor": first_payload["next_cursor"], + "sort_by": "time", + "sort_direction": "desc", + "filter_action": "system_settings", + }, + ) + self.assertEqual(second_page_full_delta.status_code, 200, second_page_full_delta.text) + full_delta_payload = second_page_full_delta.json() + self.assertTrue(full_delta_payload["full"]) + self.assertEqual([item["id"] for item in full_delta_payload["items"]], second_page_ids) + + time.sleep(0.01) + updated = self.client.patch( + "/api/v1/admin/system/settings", + headers=headers, + json={ + "default_locale": "en-cursor", + "allow_tenant_custom_groups": True, + "allow_tenant_custom_roles": True, + "allow_tenant_api_keys": True, + }, + ) + self.assertEqual(updated.status_code, 200, updated.text) + + second_page_after_insert = self.client.get( + "/api/v1/admin/audit", + headers=headers, + params={ + "scope": "system", + "page": 2, + "page_size": 2, + "cursor": first_payload["next_cursor"], + "sort_by": "time", + "sort_direction": "desc", + "filter_action": "system_settings", + }, + ) + self.assertEqual(second_page_after_insert.status_code, 200, second_page_after_insert.text) + self.assertEqual([item["id"] for item in second_page_after_insert.json()["items"]], second_page_ids) + + cursor_delta = self.client.get( + "/api/v1/admin/audit/delta", + headers=headers, + params={ + "scope": "system", + "page_size": 2, + "cursor": first_payload["next_cursor"], + "sort_by": "time", + "sort_direction": "desc", + "filter_action": "system_settings", + "since": full_delta_payload["watermark"], + }, + ) + self.assertEqual(cursor_delta.status_code, 200, cursor_delta.text) + cursor_delta_payload = cursor_delta.json() + self.assertFalse(cursor_delta_payload["full"]) + self.assertEqual(cursor_delta_payload["items"], []) + def test_tenant_owner_selection_audit_scope_and_last_owner_protection(self) -> None: headers, _ = self._login() diff --git a/tests/test_change_sequence.py b/tests/test_change_sequence.py new file mode 100644 index 0000000..69f323e --- /dev/null +++ b/tests/test_change_sequence.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_core.core.change_sequence import ( + ChangeSequenceEntry, + ChangeSequenceRetentionFloor, + decode_sequence_watermark, + encode_sequence_watermark, + max_sequence_id, + min_sequence_id, + prune_sequence_entries, + record_change, + retained_sequence_floor, + sequence_entries_since, + sequence_watermark_is_expired, +) +from govoplan_core.db.base import Base + + +class ChangeSequenceTests(unittest.TestCase): + def test_records_changes_and_returns_entries_after_watermark(self) -> None: + engine = create_engine("sqlite:///:memory:") + SessionLocal = sessionmaker(bind=engine) + try: + Base.metadata.create_all(bind=engine, tables=[ChangeSequenceEntry.__table__, ChangeSequenceRetentionFloor.__table__]) + with SessionLocal() as session: + first = record_change( + session, + tenant_id="tenant-1", + module_id="files", + collection="files.assets", + resource_type="file", + resource_id="file-1", + operation="created", + ) + second = record_change( + session, + tenant_id="tenant-1", + module_id="files", + collection="files.assets", + resource_type="file", + resource_id="file-1", + operation="updated", + ) + session.commit() + + self.assertEqual(encode_sequence_watermark(first.id), "seq:1") + self.assertEqual(decode_sequence_watermark("seq:1"), 1) + self.assertEqual(max_sequence_id(session, tenant_id="tenant-1", module_id="files"), second.id) + self.assertEqual(min_sequence_id(session, tenant_id="tenant-1", module_id="files"), first.id) + self.assertFalse(sequence_watermark_is_expired(session, tenant_id="tenant-1", module_id="files", since=first.id - 1)) + self.assertEqual( + [entry.operation for entry in sequence_entries_since(session, tenant_id="tenant-1", module_id="files", since=first.id)], + ["updated"], + ) + + session.delete(first) + session.commit() + self.assertFalse(sequence_watermark_is_expired(session, tenant_id="tenant-1", module_id="files", since=0)) + finally: + engine.dispose() + + def test_pruning_records_retention_floor_for_stale_watermarks(self) -> None: + engine = create_engine("sqlite:///:memory:") + SessionLocal = sessionmaker(bind=engine) + try: + Base.metadata.create_all(bind=engine, tables=[ChangeSequenceEntry.__table__, ChangeSequenceRetentionFloor.__table__]) + with SessionLocal() as session: + first = record_change( + session, + tenant_id="tenant-1", + module_id="files", + collection="files.assets", + resource_type="file", + resource_id="file-1", + operation="created", + ) + second = record_change( + session, + tenant_id="tenant-1", + module_id="files", + collection="files.assets", + resource_type="file", + resource_id="file-2", + operation="created", + ) + session.commit() + first_id = first.id + second_resource_id = second.resource_id + + deleted = prune_sequence_entries( + session, + tenant_id="tenant-1", + module_id="files", + collections=("files.assets",), + before_or_equal_id=first_id, + ) + session.commit() + + self.assertEqual(deleted, 1) + self.assertEqual(retained_sequence_floor(session, tenant_id="tenant-1", module_id="files", collections=("files.assets",)), first_id) + self.assertTrue(sequence_watermark_is_expired(session, tenant_id="tenant-1", module_id="files", collections=("files.assets",), since=0)) + self.assertFalse(sequence_watermark_is_expired(session, tenant_id="tenant-1", module_id="files", collections=("files.assets",), since=first_id)) + self.assertEqual( + [entry.resource_id for entry in sequence_entries_since(session, tenant_id="tenant-1", module_id="files", collections=("files.assets",), since=first_id)], + [second_resource_id], + ) + finally: + engine.dispose() + + def test_rejects_invalid_watermark(self) -> None: + with self.assertRaises(ValueError): + decode_sequence_watermark("seq:not-a-number") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_conditional_requests.py b/tests/test_conditional_requests.py new file mode 100644 index 0000000..45aecec --- /dev/null +++ b/tests/test_conditional_requests.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import unittest + +from fastapi import APIRouter, Response +from fastapi.responses import PlainTextResponse +from fastapi.testclient import TestClient + +from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.server.fastapi import create_govoplan_app +from govoplan_core.server.platform import create_platform_router + + +class ConditionalRequestTests(unittest.TestCase): + def _client(self) -> TestClient: + router = APIRouter() + + @router.get("/json") + def json_payload(value: str = "alpha") -> dict[str, str]: + return {"value": value} + + @router.get("/cookie") + def cookie_payload(response: Response) -> dict[str, bool]: + response.set_cookie("govoplan-test", "changed") + return {"ok": True} + + @router.get("/text") + def text_payload() -> PlainTextResponse: + return PlainTextResponse("plain") + + app = create_govoplan_app( + title="conditional request test", + version="test", + registry=PlatformRegistry(), + api_router=router, + ) + return TestClient(app) + + def test_json_get_receives_private_etag_and_matching_request_returns_304(self) -> None: + with self._client() as client: + first = client.get("/json", headers={"X-Request-ID": "request-1"}) + self.assertEqual(200, first.status_code, first.text) + self.assertEqual({"value": "alpha"}, first.json()) + etag = first.headers.get("etag") + self.assertIsNotNone(etag) + self.assertTrue(etag.startswith('W/"sha256-'), etag) + self.assertIn("private", first.headers.get("cache-control", "")) + self.assertIn("no-cache", first.headers.get("cache-control", "")) + self.assertIn("authorization", first.headers.get("vary", "").lower()) + self.assertEqual("request-1", first.headers["X-Correlation-ID"]) + + second = client.get("/json", headers={"If-None-Match": etag or "", "X-Request-ID": "request-2"}) + self.assertEqual(304, second.status_code, second.text) + self.assertEqual(b"", second.content) + self.assertEqual(etag, second.headers.get("etag")) + self.assertEqual("request-2", second.headers["X-Correlation-ID"]) + + def test_changed_json_body_does_not_match_previous_etag(self) -> None: + with self._client() as client: + first = client.get("/json?value=alpha") + etag = first.headers.get("etag") + self.assertIsNotNone(etag) + + changed = client.get("/json?value=beta", headers={"If-None-Match": etag or ""}) + self.assertEqual(200, changed.status_code, changed.text) + self.assertEqual({"value": "beta"}, changed.json()) + self.assertNotEqual(etag, changed.headers.get("etag")) + + def test_conditional_middleware_skips_cookie_and_non_json_responses(self) -> None: + with self._client() as client: + cookie = client.get("/cookie") + self.assertEqual(200, cookie.status_code, cookie.text) + self.assertNotIn("etag", cookie.headers) + + text = client.get("/text") + self.assertEqual(200, text.status_code, text.text) + self.assertNotIn("etag", text.headers) + + def test_platform_endpoints_return_304_for_matching_if_none_match(self) -> None: + api_router = APIRouter(prefix="/api/v1") + api_router.include_router(create_platform_router()) + app = create_govoplan_app( + title="conditional platform endpoint test", + version="test", + registry=PlatformRegistry(), + api_router=api_router, + ) + + with TestClient(app) as client: + for path in ("/api/v1/platform/status", "/api/v1/platform/modules"): + first = client.get(path) + self.assertEqual(200, first.status_code, first.text) + etag = first.headers.get("etag") + self.assertIsNotNone(etag, path) + + second = client.get(path, headers={"If-None-Match": etag or ""}) + self.assertEqual(304, second.status_code, second.text) + self.assertEqual(b"", second.content) + self.assertEqual(etag, second.headers.get("etag")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_core_events.py b/tests/test_core_events.py new file mode 100644 index 0000000..b96f881 --- /dev/null +++ b/tests/test_core_events.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import shutil +import tempfile +import unittest +from pathlib import Path + +from fastapi import APIRouter +from fastapi.testclient import TestClient + +from govoplan_core.audit.logging import audit_event, audit_operation_context +from govoplan_core.core.events import EventBus, PlatformEvent, current_event_trace, event_context, normalize_trace_id +from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.base import Base +from govoplan_core.server.fastapi import create_govoplan_app +from tests.db_isolation import temporary_database + + +class CoreEventTests(unittest.TestCase): + def test_event_bus_adds_trace_ids_and_propagates_causation_to_nested_events(self) -> None: + bus = EventBus() + seen: list[PlatformEvent] = [] + + def parent_handler(event: PlatformEvent) -> None: + seen.append(event) + bus.publish(PlatformEvent(type="child.ready", module_id="demo")) + + bus.subscribe("parent.ready", parent_handler) + bus.subscribe("child.ready", seen.append) + + bus.publish(PlatformEvent(type="parent.ready", module_id="demo", correlation_id="corr-1")) + + self.assertEqual(2, len(seen)) + parent, child = seen + self.assertEqual("corr-1", parent.correlation_id) + self.assertIsNone(parent.causation_id) + self.assertEqual("corr-1", child.correlation_id) + self.assertEqual(parent.event_id, child.causation_id) + + def test_request_correlation_middleware_sets_event_context_and_response_header(self) -> None: + router = APIRouter() + + @router.get("/trace") + def trace() -> dict[str, str | None]: + active = current_event_trace() + return {"correlation_id": active.correlation_id if active else None} + + app = create_govoplan_app( + title="event test", + version="test", + registry=PlatformRegistry(), + api_router=router, + ) + + with TestClient(app) as client: + response = client.get("/trace", headers={"X-Request-ID": "request-123"}) + + self.assertEqual(200, response.status_code, response.text) + self.assertEqual("request-123", response.json()["correlation_id"]) + self.assertEqual("request-123", response.headers["X-Correlation-ID"]) + + def test_audit_event_persists_trace_details_from_event_context(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-audit-trace-")) + try: + with temporary_database(f"sqlite:///{root / 'audit.db'}") as database: + from govoplan_access.backend.db import models as access_models # noqa: F401 + from govoplan_admin.backend.db import models as admin_models # noqa: F401 + from govoplan_audit.backend.db import models as audit_models # noqa: F401 + from govoplan_tenancy.backend.db import models as tenancy_models # noqa: F401 + + Base.metadata.create_all(bind=database.engine) + with database.session() as session: + with event_context(correlation_id="corr-1", causation_id="cause-1"): + item = audit_event( + session, + tenant_id=None, + scope="system", + action="demo.changed", + object_type="demo", + object_id="demo-1", + details={"password": "secret"}, + ) + + self.assertEqual({"correlation_id": "corr-1", "causation_id": "cause-1"}, item.details["_trace"]) + self.assertEqual("", item.details["password"]) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_audit_operation_context_keeps_lifecycle_ids_and_sanitizes_details(self) -> None: + payload = audit_operation_context( + module_id="mail", + request_id="request-1", + run_id="run-1", + outcome="queued", + trace={"correlation_id": " corr-1 ", "causation_id": "cause-1", "token": "secret"}, + options={"migrate_database": True, "api_key": "secret"}, + empty=None, + ) + + self.assertEqual("mail", payload["module_id"]) + self.assertEqual("request-1", payload["request_id"]) + self.assertEqual("run-1", payload["run_id"]) + self.assertEqual("queued", payload["outcome"]) + self.assertEqual({"correlation_id": "corr-1", "causation_id": "cause-1"}, payload["_trace"]) + self.assertEqual("", payload["options"]["api_key"]) + self.assertNotIn("token", payload["_trace"]) + self.assertNotIn("empty", payload) + + def test_trace_ids_are_normalized_for_headers_and_audit_input(self) -> None: + self.assertEqual("abc-123", normalize_trace_id(" abc-123 ")) + self.assertIsNone(normalize_trace_id("../bad")) + self.assertIsNone(normalize_trace_id("x" * 129)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_database_migrations.py b/tests/test_database_migrations.py index 5970afc..892bc65 100644 --- a/tests/test_database_migrations.py +++ b/tests/test_database_migrations.py @@ -12,10 +12,13 @@ from sqlalchemy import create_engine, inspect, text from govoplan_core.db.base import Base from govoplan_core.db.migrations import ( REVISION_AUTH_RBAC, + REVISION_CORE_CHANGE_SEQUENCE, REVISION_FILE_FOLDERS, REVISION_HIERARCHICAL_SETTINGS, alembic_config, migrate_database, + reconcile_change_sequence_retention_floor_drift, + reconcile_namespace_table_drift, ) @@ -28,6 +31,67 @@ def database_migration_heads(connection) -> set[str]: class DatabaseMigrationTests(unittest.TestCase): + def test_repairs_missing_change_sequence_retention_floor(self) -> None: + with tempfile.TemporaryDirectory(prefix="msm-change-sequence-drift-test-") as directory: + database = Path(directory) / "change-sequence-drift.db" + url = f"sqlite:///{database}" + command.upgrade(alembic_config(database_url=url, enabled_modules=()), REVISION_CORE_CHANGE_SEQUENCE) + + engine = create_engine(url) + try: + with engine.begin() as connection: + connection.execute(text("DROP TABLE core_change_sequence_retention_floor")) + with engine.connect() as connection: + tables = set(inspect(connection).get_table_names()) + self.assertIn("core_change_sequence", tables) + self.assertNotIn("core_change_sequence_retention_floor", tables) + finally: + engine.dispose() + + self.assertTrue(reconcile_change_sequence_retention_floor_drift(url)) + + engine = create_engine(url) + try: + with engine.connect() as connection: + inspector = inspect(connection) + self.assertIn("core_change_sequence_retention_floor", inspector.get_table_names()) + self.assertIn( + "ix_core_change_sequence_retention_scope", + {index["name"] for index in inspector.get_indexes("core_change_sequence_retention_floor")}, + ) + finally: + engine.dispose() + + def test_repairs_namespace_tables_when_database_was_stamped_ahead(self) -> None: + with tempfile.TemporaryDirectory(prefix="msm-namespace-drift-test-") as directory: + database = Path(directory) / "namespace-drift.db" + url = f"sqlite:///{database}" + engine = create_engine(url) + try: + with engine.begin() as connection: + connection.execute(text("CREATE TABLE accounts (id TEXT PRIMARY KEY)")) + connection.execute(text("CREATE TABLE users (id TEXT PRIMARY KEY)")) + connection.execute(text("CREATE TABLE governance_templates (id TEXT PRIMARY KEY)")) + with engine.connect() as connection: + tables = set(inspect(connection).get_table_names()) + self.assertIn("accounts", tables) + self.assertNotIn("access_accounts", tables) + finally: + engine.dispose() + + self.assertTrue(reconcile_namespace_table_drift(url)) + + engine = create_engine(url) + try: + with engine.connect() as connection: + tables = set(inspect(connection).get_table_names()) + self.assertNotIn("accounts", tables) + self.assertIn("access_accounts", tables) + self.assertNotIn("governance_templates", tables) + self.assertIn("admin_governance_templates", tables) + finally: + engine.dispose() + def test_repairs_create_all_schema_drift_and_upgrades_to_head(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-migration-test-") as directory: database = Path(directory) / "legacy.db" @@ -72,33 +136,33 @@ class DatabaseMigrationTests(unittest.TestCase): job_indexes = {index["name"] for index in inspector.get_indexes("campaign_jobs")} attempt_columns = {column["name"] for column in inspector.get_columns("send_attempts")} tables = set(inspector.get_table_names()) - tenant_columns = {column["name"] for column in inspector.get_columns("tenants")} - user_columns = {column["name"] for column in inspector.get_columns("users")} - session_columns = {column["name"] for column in inspector.get_columns("auth_sessions")} - group_columns = {column["name"] for column in inspector.get_columns("groups")} - role_columns = {column["name"] for column in inspector.get_columns("roles")} - role_indexes = {index["name"] for index in inspector.get_indexes("roles")} + tenant_columns = {column["name"] for column in inspector.get_columns("tenancy_tenants")} + user_columns = {column["name"] for column in inspector.get_columns("access_users")} + session_columns = {column["name"] for column in inspector.get_columns("access_auth_sessions")} + group_columns = {column["name"] for column in inspector.get_columns("access_groups")} + role_columns = {column["name"] for column in inspector.get_columns("access_roles")} + role_indexes = {index["name"] for index in inspector.get_indexes("access_roles")} campaign_columns = {column["name"] for column in inspector.get_columns("campaigns")} audit_columns = {column["name"] for column in inspector.get_columns("audit_log")} audit_indexes = {index["name"] for index in inspector.get_indexes("audit_log")} system_role_flags = { row["slug"]: bool(row["is_builtin"]) for row in connection.execute(text( - "SELECT slug, is_builtin FROM roles WHERE tenant_id IS NULL AND slug IN ('system_owner', 'system_admin', 'system_auditor')" + "SELECT slug, is_builtin FROM access_roles WHERE tenant_id IS NULL AND slug IN ('system_owner', 'system_admin', 'system_auditor')" )).mappings().all() } - system_settings_columns = {column["name"] for column in inspector.get_columns("system_settings")} - governance_assignment_columns = {column["name"] for column in inspector.get_columns("governance_template_assignments")} + system_settings_columns = {column["name"] for column in inspector.get_columns("core_system_settings")} + governance_assignment_columns = {column["name"] for column in inspector.get_columns("admin_governance_template_assignments")} import_profile_columns = {column["name"] for column in inspector.get_columns("campaign_recipient_import_mapping_profiles")} import_profile_indexes = {index["name"] for index in inspector.get_indexes("campaign_recipient_import_mapping_profiles")} - account_count = connection.execute(text("SELECT COUNT(*) FROM accounts")).scalar_one() - user_count = connection.execute(text("SELECT COUNT(*) FROM users")).scalar_one() + account_count = connection.execute(text("SELECT COUNT(*) FROM access_accounts")).scalar_one() + user_count = connection.execute(text("SELECT COUNT(*) FROM access_users")).scalar_one() system_owner_count = connection.execute(text( """ SELECT COUNT(*) - FROM system_role_assignments assignment - JOIN roles role ON role.id = assignment.role_id - JOIN accounts account ON account.id = assignment.account_id + FROM access_system_role_assignments assignment + JOIN access_roles role ON role.id = assignment.role_id + JOIN access_accounts account ON account.id = assignment.account_id WHERE role.slug = 'system_owner' AND account.is_active = 1 """ )).scalar_one() @@ -120,11 +184,11 @@ class DatabaseMigrationTests(unittest.TestCase): self.assertIn("ix_campaign_jobs_eml_sha256", job_indexes) self.assertIn("status", attempt_columns) self.assertIn("claim_token", attempt_columns) - self.assertIn("accounts", tables) - self.assertIn("system_role_assignments", tables) - self.assertIn("system_settings", tables) - self.assertIn("governance_templates", tables) - self.assertIn("governance_template_assignments", tables) + self.assertIn("access_accounts", tables) + self.assertIn("access_system_role_assignments", tables) + self.assertIn("core_system_settings", tables) + self.assertIn("admin_governance_templates", tables) + self.assertIn("admin_governance_template_assignments", tables) self.assertIn("campaign_shares", tables) self.assertIn("campaign_recipient_import_mapping_profiles", tables) self.assertIn("owner_user_id", campaign_columns) @@ -259,13 +323,13 @@ class DatabaseMigrationTests(unittest.TestCase): timestamps = {"created_at": "2026-06-14 12:00:00", "updated_at": "2026-06-14 12:00:00"} connection.execute(text( """ - INSERT INTO tenants (id, slug, name, is_active, created_at, updated_at) + INSERT INTO tenancy_tenants (id, slug, name, is_active, created_at, updated_at) VALUES ('tenant-1', 'legacy', 'Legacy', 1, :created_at, :updated_at) """ ), timestamps) connection.execute(text( """ - INSERT INTO users + INSERT INTO access_users (id, tenant_id, email, display_name, is_active, is_tenant_admin, auth_provider, password_hash, last_login_at, created_at, updated_at) VALUES @@ -275,7 +339,7 @@ class DatabaseMigrationTests(unittest.TestCase): ), timestamps) connection.execute(text( """ - INSERT INTO auth_sessions + INSERT INTO access_auth_sessions (id, tenant_id, user_id, token_hash, expires_at, last_seen_at, revoked_at, user_agent, ip_address, created_at, updated_at) VALUES @@ -292,37 +356,37 @@ class DatabaseMigrationTests(unittest.TestCase): try: with engine.connect() as connection: account = connection.execute(text( - "SELECT id, normalized_email, password_hash FROM accounts" + "SELECT id, normalized_email, password_hash FROM access_accounts" )).mappings().one() membership = connection.execute(text( - "SELECT account_id FROM users WHERE id = 'user-1'" + "SELECT account_id FROM access_users WHERE id = 'user-1'" )).mappings().one() migrated_session = connection.execute(text( - "SELECT account_id FROM auth_sessions WHERE id = 'session-1'" + "SELECT account_id FROM access_auth_sessions WHERE id = 'session-1'" )).mappings().one() owner_assignment = connection.execute(text( """ SELECT role.slug - FROM user_role_assignments assignment - JOIN roles role ON role.id = assignment.role_id + FROM access_user_role_assignments assignment + JOIN access_roles role ON role.id = assignment.role_id WHERE assignment.user_id = 'user-1' """ )).scalar_one() owner_permissions = connection.execute(text( - "SELECT permissions FROM roles WHERE tenant_id = 'tenant-1' AND slug = 'owner'" + "SELECT permissions FROM access_roles WHERE tenant_id = 'tenant-1' AND slug = 'owner'" )).scalar_one() system_assignment = connection.execute(text( """ SELECT role.slug - FROM system_role_assignments assignment - JOIN roles role ON role.id = assignment.role_id + FROM access_system_role_assignments assignment + JOIN access_roles role ON role.id = assignment.role_id WHERE assignment.account_id = :account_id """ ), {"account_id": account["id"]}).scalar_one() system_role_flags = { row["slug"]: bool(row["is_builtin"]) for row in connection.execute(text( - "SELECT slug, is_builtin FROM roles WHERE tenant_id IS NULL AND slug IN ('system_owner', 'system_admin', 'system_auditor')" + "SELECT slug, is_builtin FROM access_roles WHERE tenant_id IS NULL AND slug IN ('system_owner', 'system_admin', 'system_auditor')" )).mappings().all() } self.assertEqual(account["normalized_email"], "owner@example.local") @@ -356,7 +420,7 @@ class DatabaseMigrationTests(unittest.TestCase): INSERT INTO audit_log (id, tenant_id, user_id, api_key_id, action, object_type, object_id, details, created_at, updated_at) VALUES - ('audit-system', NULL, NULL, NULL, 'system_settings.updated', 'system_settings', 'system', '{}', :created_at, :updated_at), + ('audit-system', NULL, NULL, NULL, 'system_settings.updated', 'core_system_settings', 'system', '{}', :created_at, :updated_at), ('audit-tenant', NULL, NULL, NULL, 'user.updated', 'user', 'user-1', '{}', :created_at, :updated_at) """ ), timestamps) diff --git a/tests/test_devserver.py b/tests/test_devserver.py index 3dbbde6..18cc4ad 100644 --- a/tests/test_devserver.py +++ b/tests/test_devserver.py @@ -18,6 +18,10 @@ class DevserverSmokeTests(unittest.TestCase): for key in ( "DATABASE_URL", "DEV_BOOTSTRAP_ENABLED", + "GOVOPLAN_DEV_DATABASE_BACKEND", + "GOVOPLAN_DEV_DATABASE_URL", + "GOVOPLAN_DEV_DATABASE_URL_PGTOOLS", + "GOVOPLAN_DATABASE_URL_PGTOOLS", "GOVOPLAN_SERVER_CONFIG", "FILE_STORAGE_LOCAL_ROOT", "MOCK_MAILBOX_DIR", @@ -26,6 +30,7 @@ class DevserverSmokeTests(unittest.TestCase): env["APP_ENV"] = "dev" env["CELERY_ENABLED"] = "false" env["ENABLED_MODULES"] = "access" + env["GOVOPLAN_DEV_DATABASE_BACKEND"] = "sqlite" env["PYTHONPATH"] = str(src_root) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "") result = subprocess.run( diff --git a/tests/test_identity_organization_contracts.py b/tests/test_identity_organization_contracts.py new file mode 100644 index 0000000..fb2d446 --- /dev/null +++ b/tests/test_identity_organization_contracts.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import shutil +import tempfile +import unittest +from pathlib import Path + +from govoplan_core.core.identity import ( + CAPABILITY_IDENTITY_DIRECTORY, + IdentityAccountLinkRef, + IdentityDirectory, + IdentityRef, +) +from govoplan_core.core.organizations import ( + CAPABILITY_ORGANIZATION_DIRECTORY, + OrganizationDirectory, + OrganizationFunctionAssignmentRef, + OrganizationFunctionRef, + OrganizationUnitRef, +) +from govoplan_core.core.modules import ModuleContext, ModuleManifest +from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.base import Base +from govoplan_access.backend.db.models import User # noqa: F401 - registers legacy tenancy relationship target +from govoplan_identity.backend.db.models import Identity, IdentityAccountLink +from govoplan_organizations.backend.db.models import ( + OrganizationFunction, + OrganizationFunctionAssignment, + OrganizationUnit, +) +from govoplan_tenancy.backend.db.models import Tenant +from tests.db_isolation import temporary_database + + +class _FakeIdentityDirectory: + identity = IdentityRef( + id="identity-1", + display_name="Demo Person", + primary_account_id="account-1", + account_ids=("account-1", "account-2"), + ) + + def get_identity(self, identity_id: str) -> IdentityRef | None: + return self.identity if identity_id == self.identity.id else None + + def identity_for_account(self, account_id: str) -> IdentityRef | None: + return self.identity if account_id in self.identity.account_ids else None + + def identities_for_accounts(self, account_ids): + return (self.identity,) if set(account_ids) & set(self.identity.account_ids) else () + + def accounts_for_identity(self, identity_id: str): + if identity_id != self.identity.id: + return () + return ( + IdentityAccountLinkRef(id="link-1", identity_id=identity_id, account_id="account-1", is_primary=True), + IdentityAccountLinkRef(id="link-2", identity_id=identity_id, account_id="account-2"), + ) + + +class _FakeOrganizationDirectory: + unit = OrganizationUnitRef(id="ou-1", tenant_id="tenant-1", slug="registry", name="Registry") + function = OrganizationFunctionRef( + id="function-1", + tenant_id="tenant-1", + organization_unit_id=unit.id, + slug="registry-clerk", + name="Registry Clerk", + ) + assignment = OrganizationFunctionAssignmentRef( + id="assignment-1", + tenant_id="tenant-1", + account_id="account-1", + identity_id="identity-1", + function_id=function.id, + organization_unit_id=unit.id, + applies_to_subunits=True, + ) + + def get_organization_unit(self, organization_unit_id: str) -> OrganizationUnitRef | None: + return self.unit if organization_unit_id == self.unit.id else None + + def organization_units_for_tenant(self, tenant_id: str): + return (self.unit,) if tenant_id == self.unit.tenant_id else () + + def get_function(self, function_id: str) -> OrganizationFunctionRef | None: + return self.function if function_id == self.function.id else None + + def functions_for_organization_unit(self, organization_unit_id: str, *, include_subunits: bool = False): + del include_subunits + return (self.function,) if organization_unit_id == self.unit.id else () + + def get_function_assignment(self, assignment_id: str) -> OrganizationFunctionAssignmentRef | None: + return self.assignment if assignment_id == self.assignment.id else None + + def function_assignments_for_account(self, account_id: str, *, tenant_id: str | None = None): + if account_id != self.assignment.account_id: + return () + if tenant_id is not None and tenant_id != self.assignment.tenant_id: + return () + return (self.assignment,) + + +class IdentityOrganizationContractTests(unittest.TestCase): + def test_protocol_shapes_match_reference_implementations(self) -> None: + self.assertIsInstance(_FakeIdentityDirectory(), IdentityDirectory) + self.assertIsInstance(_FakeOrganizationDirectory(), OrganizationDirectory) + + def test_capabilities_register_and_resolve_through_platform_registry(self) -> None: + identity_directory = _FakeIdentityDirectory() + organization_directory = _FakeOrganizationDirectory() + manifest = ModuleManifest( + id="directory-contract-test", + name="Directory Contract Test", + version="test", + capability_factories={ + CAPABILITY_IDENTITY_DIRECTORY: lambda context: identity_directory, + CAPABILITY_ORGANIZATION_DIRECTORY: lambda context: organization_directory, + }, + ) + registry = PlatformRegistry() + registry.register(manifest) + registry.configure_capability_context(ModuleContext(registry=registry, settings=object())) + + self.assertIs(identity_directory, registry.require_capability(CAPABILITY_IDENTITY_DIRECTORY)) + self.assertIs(organization_directory, registry.require_capability(CAPABILITY_ORGANIZATION_DIRECTORY)) + + def test_sql_identity_and_organization_directories_return_dtos(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-directory-contracts-")) + try: + with temporary_database(f"sqlite:///{root / 'directory.db'}") as database: + Base.metadata.create_all(bind=database.engine) + with database.session() as session: + tenant = Tenant(id="tenant-directory", slug="directory", name="Directory Tenant", settings={}) + identity = Identity(id="identity-directory", display_name="Directory Person", source="local") + link = IdentityAccountLink( + identity_id=identity.id, + account_id="account-directory", + is_primary=True, + ) + unit = OrganizationUnit( + id="ou-directory", + tenant_id=tenant.id, + slug="registry", + name="Registry", + ) + function = OrganizationFunction( + id="function-directory", + tenant_id=tenant.id, + organization_unit_id=unit.id, + slug="registry-clerk", + name="Registry Clerk", + ) + assignment = OrganizationFunctionAssignment( + id="assignment-directory", + tenant_id=tenant.id, + account_id="account-directory", + identity_id=identity.id, + function_id=function.id, + organization_unit_id=unit.id, + applies_to_subunits=True, + ) + session.add_all([tenant, identity, link, unit, function, assignment]) + session.commit() + + from govoplan_identity.backend.directory import SqlIdentityDirectory + from govoplan_organizations.backend.directory import SqlOrganizationDirectory + + identity_directory = SqlIdentityDirectory() + organization_directory = SqlOrganizationDirectory() + + self.assertEqual("Directory Person", identity_directory.get_identity("identity-directory").display_name) # type: ignore[union-attr] + self.assertEqual("identity-directory", identity_directory.identity_for_account("account-directory").id) # type: ignore[union-attr] + self.assertEqual(["account-directory"], [item.account_id for item in identity_directory.accounts_for_identity("identity-directory")]) + self.assertEqual("Registry", organization_directory.get_organization_unit("ou-directory").name) # type: ignore[union-attr] + self.assertEqual("Registry Clerk", organization_directory.get_function("function-directory").name) # type: ignore[union-attr] + self.assertEqual( + ["assignment-directory"], + [item.id for item in organization_directory.function_assignments_for_account("account-directory", tenant_id="tenant-directory")], + ) + finally: + shutil.rmtree(root, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_module_system.py b/tests/test_module_system.py index 9b01a5f..ae321f3 100644 --- a/tests/test_module_system.py +++ b/tests/test_module_system.py @@ -1,9 +1,11 @@ from __future__ import annotations +import hashlib import importlib import base64 import json import os +import re import shlex import shutil import subprocess @@ -12,6 +14,7 @@ import tempfile import textwrap import tomllib import unittest +import urllib.error from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -26,10 +29,12 @@ os.environ.setdefault("DATABASE_URL", f"sqlite:///{_TEST_ROOT / 'module-test.db' os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false") os.environ.setdefault("CELERY_ENABLED", "false") +from fastapi import APIRouter from fastapi.testclient import TestClient from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from govoplan_core.core.events import event_context from govoplan_core.core.migrations import migration_metadata_plan from govoplan_core.core.module_management import ( ModuleInstallPlan, @@ -63,24 +68,45 @@ from govoplan_core.core.module_installer import ( update_module_installer_daemon_status, update_module_installer_request, ) -from govoplan_core.core.module_license import module_license_decision, validate_module_license +from govoplan_core.core.configuration_packages import ( + CONFIGURATION_PROVIDER_CAPABILITY, + ConfigurationApplyResult, + ConfigurationExportResult, + ConfigurationExportSelection, + ConfigurationPackageFragment, + ConfigurationPlanItem, + ConfigurationPreflightContext, + ConfigurationPreflightResult, + ConfigurationProvider, + ConfigurationProviderDescription, + ConfigurationRequiredData, + apply_configuration_package, + configuration_package_catalog, + dry_run_configuration_package, + export_configuration_package, + sign_configuration_package_catalog, + validate_configuration_package_catalog, +) +from govoplan_core.core.module_license import issue_module_license, module_license_decision, module_license_diagnostics, validate_module_license from govoplan_core.core.module_package_catalog import ( module_package_catalog, record_module_package_catalog_acceptance, sign_module_package_catalog, validate_module_package_catalog, ) -from govoplan_core.core.modules import MigrationRetirementPlan, ModuleCompatibility, ModuleUninstallGuardResult +from govoplan_core.core.modules import FrontendModule, FrontendRoute, MigrationRetirementPlan, ModuleCompatibility, ModuleUninstallGuardResult from govoplan_core.core.module_guards import drop_table_retirement_provider from govoplan_core.core.modules import MigrationSpec, ModuleManifest +from govoplan_core.core.registry import PlatformRegistry, RegistryError from govoplan_core.admin.models import SystemSettings from govoplan_core.db.base import Base -from govoplan_core.db.session import configure_database, get_database +from govoplan_core.db.session import configure_database as configure_database_handle, get_database, reset_database from govoplan_core.security.module_permissions import scopes_grant_compatible from govoplan_core.security.permissions import scope_grants from govoplan_core.server.app import create_app from govoplan_core.server.config import GovoplanServerConfig from govoplan_core.server.registry import available_module_manifests, build_platform_registry +from govoplan_core.server.route_validation import RouteCollisionError from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend, StorageBackendError _MANIFEST_PROJECT_MODULES = { @@ -89,10 +115,45 @@ _MANIFEST_PROJECT_MODULES = { "tenancy": "govoplan_tenancy", "policy": "govoplan_policy", "audit": "govoplan_audit", + "dashboard": "govoplan_dashboard", "files": "govoplan_files", "mail": "govoplan_mail", "campaigns": "govoplan_campaign", + "docs": "govoplan_docs", + "ops": "govoplan_ops", } +_MODULE_ID_RE = re.compile(r"^[a-z][a-z0-9_]*$") +_PACKAGE_NAME_RE = re.compile(r"^@govoplan/[a-z0-9_-]+-webui$") + + +def configure_database(database_url: str): + return configure_database_handle(database_url, dispose_previous=True) + + +def _join_route_path(prefix: str, path: str) -> str: + if not prefix: + return path + if not path: + return prefix + return f"{prefix.rstrip('/')}/{path.lstrip('/')}" + + +def _route_paths(routes_or_app, *, prefix: str = "") -> set[str]: + routes = getattr(routes_or_app, "routes", routes_or_app) + paths: set[str] = set() + for route in routes: + path = getattr(route, "path", None) + if path: + paths.add(_join_route_path(prefix, path)) + include_context = getattr(route, "include_context", None) + nested_prefix = _join_route_path(prefix, str(getattr(include_context, "prefix", "") or "")) + nested_router = getattr(route, "original_router", None) + if nested_router is not None: + paths.update(_route_paths(nested_router, prefix=nested_prefix)) + nested_routes = getattr(route, "routes", None) + if nested_routes is not None: + paths.update(_route_paths(nested_routes, prefix=nested_prefix)) + return paths def _settings(root: Path) -> SimpleNamespace: @@ -124,6 +185,7 @@ def _settings(root: Path) -> SimpleNamespace: class ModuleSystemTests(unittest.TestCase): @classmethod def tearDownClass(cls) -> None: + reset_database(dispose=True) shutil.rmtree(_TEST_ROOT, ignore_errors=True) def _app_for_modules(self, modules: tuple[str, ...]): @@ -156,9 +218,12 @@ class ModuleSystemTests(unittest.TestCase): def test_discovers_installed_core_and_product_module_manifests(self) -> None: manifests = available_module_manifests() - self.assertTrue({"access", "admin", "tenancy", "policy", "audit", "files", "mail", "campaigns"}.issubset(manifests)) + self.assertTrue({"access", "admin", "tenancy", "policy", "audit", "dashboard", "files", "mail", "campaigns", "docs", "ops"}.issubset(manifests)) self.assertEqual(manifests["campaigns"].dependencies, ("access",)) self.assertEqual(manifests["campaigns"].optional_dependencies, ("files", "mail")) + self.assertEqual(manifests["dashboard"].dependencies, ("access",)) + self.assertEqual(manifests["docs"].dependencies, ("access",)) + self.assertEqual(manifests["ops"].dependencies, ("access",)) def test_access_manifest_contributes_admin_frontend_metadata(self) -> None: manifests = available_module_manifests() @@ -173,6 +238,22 @@ class ModuleSystemTests(unittest.TestCase): self.assertTrue(access.frontend.routes[0].required_any) self.assertEqual("admin", access.frontend.nav_items[0].icon) + def test_obsolete_access_core_import_aliases_are_removed(self) -> None: + obsolete_paths = ( + "govoplan_core.security.api_keys", + "govoplan_core.security.sessions", + "govoplan_core.security.passwords", + "govoplan_core.api.v1.auth", + "govoplan_core.api.v1.admin", + "govoplan_core.api.v1.admin_schemas", + "govoplan_core.admin.service", + "govoplan_core.admin.governance", + ) + + for legacy_path in obsolete_paths: + with self.subTest(legacy_path=legacy_path): + self.assertIsNone(importlib.util.find_spec(legacy_path)) + def test_tenant_wildcard_grants_module_tenant_scopes(self) -> None: self.assertTrue(scope_grants("tenant:*", "calendar:event:read")) self.assertTrue(scopes_grant_compatible(["tenant:*"], "calendar:event:read")) @@ -188,18 +269,18 @@ class ModuleSystemTests(unittest.TestCase): from govoplan_tenancy.backend.db.models import Tenant self.assertIs(AccessBase, Base) - self.assertEqual("accounts", Account.__tablename__) - self.assertEqual("tenants", Tenant.__tablename__) - self.assertEqual("users", User.__tablename__) - self.assertEqual("api_keys", ApiKey.__tablename__) - self.assertEqual("auth_sessions", AuthSession.__tablename__) + self.assertEqual("access_accounts", Account.__tablename__) + self.assertEqual("tenancy_tenants", Tenant.__tablename__) + self.assertEqual("access_users", User.__tablename__) + self.assertEqual("access_api_keys", ApiKey.__tablename__) + self.assertEqual("access_auth_sessions", AuthSession.__tablename__) self.assertEqual("audit_log", AuditLog.__tablename__) - self.assertEqual("governance_templates", GovernanceTemplate.__tablename__) - self.assertEqual("system_settings", SystemSettings.__tablename__) - self.assertIn("accounts", Base.metadata.tables) - self.assertIn("tenants", Base.metadata.tables) - self.assertNotIn("access_accounts", Base.metadata.tables) - self.assertNotIn("access_auth_sessions", Base.metadata.tables) + self.assertEqual("admin_governance_templates", GovernanceTemplate.__tablename__) + self.assertEqual("core_system_settings", SystemSettings.__tablename__) + self.assertIn("access_accounts", Base.metadata.tables) + self.assertIn("tenancy_tenants", Base.metadata.tables) + self.assertNotIn("accounts", Base.metadata.tables) + self.assertNotIn("auth_sessions", Base.metadata.tables) def test_module_manifest_versions_match_source_project_versions(self) -> None: manifests = available_module_manifests() @@ -208,6 +289,123 @@ class ModuleSystemTests(unittest.TestCase): self.assertIn(manifest_id, manifests) self.assertEqual(self._source_project_version(package_module), manifests[manifest_id].version) + def test_available_module_manifests_follow_contract_shape(self) -> None: + manifests = available_module_manifests() + for manifest_id, manifest in manifests.items(): + with self.subTest(manifest_id=manifest_id): + self.assertRegex(manifest.id, _MODULE_ID_RE) + self.assertEqual(manifest_id, manifest.id) + self.assertTrue(manifest.name.strip()) + self.assertTrue(manifest.version.strip()) + self.assertEqual(len(manifest.dependencies), len(set(manifest.dependencies))) + self.assertEqual(len(manifest.optional_dependencies), len(set(manifest.optional_dependencies))) + self.assertFalse(set(manifest.dependencies) & set(manifest.optional_dependencies)) + self.assertNotIn(manifest.id, manifest.dependencies) + self.assertNotIn(manifest.id, manifest.optional_dependencies) + self.assertEqual(manifest.compatibility.manifest_contract_version, "1") + if manifest.migration_spec is not None: + self.assertEqual(manifest.id, manifest.migration_spec.module_id) + self.assertTrue(manifest.migration_spec.metadata is not None or manifest.migration_spec.script_location) + if manifest.frontend is not None: + self.assertEqual(manifest.id, manifest.frontend.module_id) + if manifest.frontend.package_name is not None: + self.assertRegex(manifest.frontend.package_name, _PACKAGE_NAME_RE) + for route in (*manifest.frontend.routes, *manifest.frontend.settings_routes): + self.assertTrue(route.path.startswith("/")) + self.assertTrue(route.component.strip()) + for item in (*manifest.nav_items, *manifest.frontend.nav_items): + self.assertTrue(item.path.startswith("/")) + self.assertTrue(item.label.strip()) + if item.icon is not None: + self.assertTrue(item.icon.strip()) + + def test_registry_rejects_unsupported_manifest_contract_version(self) -> None: + registry = PlatformRegistry() + registry.register(ModuleManifest( + id="example", + name="Example", + version="test", + compatibility=ModuleCompatibility(manifest_contract_version="2"), + )) + + with self.assertRaisesRegex(RegistryError, "unsupported manifest contract version"): + registry.validate() + + def test_registry_rejects_invalid_frontend_manifest_shape(self) -> None: + registry = PlatformRegistry() + registry.register(ModuleManifest( + id="example", + name="Example", + version="test", + frontend=FrontendModule( + module_id="example", + package_name="@govoplan/example-webui", + routes=(FrontendRoute(path="missing-leading-slash", component="ExamplePage"),), + ), + )) + + with self.assertRaisesRegex(RegistryError, "Frontend route"): + registry.validate() + + def test_server_startup_rejects_duplicate_configured_routes(self) -> None: + first = APIRouter() + second = APIRouter() + + @first.get("/collision") + def first_collision() -> dict[str, bool]: + return {"first": True} + + @second.get("/collision") + def second_collision() -> dict[str, bool]: + return {"second": True} + + config = GovoplanServerConfig( + title="GovOPlaN Route Collision Test", + version="test", + settings=_settings(Path(tempfile.mkdtemp(prefix="govoplan-route-collision-", dir=_TEST_ROOT))), + enabled_modules=(), + base_routers=(first, second), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + + with self.assertRaisesRegex(RouteCollisionError, "GET .*/collision"): + create_app(config) + + def test_module_lifecycle_rejects_duplicate_module_routes(self) -> None: + def route_factory(label: str): + def factory(_context): + router = APIRouter() + + @router.get("/collision") + def collision() -> dict[str, str]: + return {"module": label} + + return router + + return factory + + config = GovoplanServerConfig( + title="GovOPlaN Module Route Collision Test", + version="test", + settings=_settings(Path(tempfile.mkdtemp(prefix="govoplan-module-route-collision-", dir=_TEST_ROOT))), + enabled_modules=("alpha", "beta"), + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + manifest_factories=( + lambda: ModuleManifest(id="alpha", name="Alpha", version="test", route_factory=route_factory("alpha")), + lambda: ModuleManifest(id="beta", name="Beta", version="test", route_factory=route_factory("beta")), + ), + ) + + with self.assertRaisesRegex(RouteCollisionError, "GET .*/collision"): + create_app(config) + def test_enabled_module_permutations_register_expected_routes(self) -> None: cases = ( ("core_only", (), {"tenancy", "access"}, set()), @@ -216,14 +414,18 @@ class ModuleSystemTests(unittest.TestCase): ("campaign_without_files_or_mail", ("campaigns",), {"tenancy", "access", "campaigns"}, {"/api/v1/campaigns"}), ("campaign_without_files", ("campaigns", "mail"), {"tenancy", "access", "campaigns", "mail"}, {"/api/v1/campaigns", "/api/v1/mail"}), ("campaign_with_files_but_no_mail", ("campaigns", "files"), {"tenancy", "access", "campaigns", "files"}, {"/api/v1/campaigns", "/api/v1/files"}), - ("full_product", ("campaigns", "files", "mail"), {"tenancy", "access", "campaigns", "files", "mail"}, {"/api/v1/campaigns", "/api/v1/files", "/api/v1/mail"}), + ("dashboard_only", ("dashboard",), {"tenancy", "access", "dashboard"}, set()), + ("docs_and_ops", ("docs", "ops"), {"tenancy", "access", "docs", "ops"}, {"/api/v1/docs", "/api/v1/ops"}), + ("full_product", ("dashboard", "campaigns", "files", "mail", "docs", "ops"), {"tenancy", "access", "dashboard", "campaigns", "files", "mail", "docs", "ops"}, {"/api/v1/campaigns", "/api/v1/files", "/api/v1/mail", "/api/v1/docs", "/api/v1/ops"}), ) for name, enabled_modules, expected_modules, expected_prefixes in cases: with self.subTest(name=name): app, _settings_obj = self._app_for_modules(enabled_modules) - route_paths = {getattr(route, "path", "") for route in app.routes} + route_paths = _route_paths(app) self.assertIn("/api/v1/platform/modules", route_paths) - for prefix in ("/api/v1/campaigns", "/api/v1/files", "/api/v1/mail"): + self.assertIn("/api/v1/auth/login", route_paths) + self.assertIn("/api/v1/admin/users", route_paths) + for prefix in ("/api/v1/campaigns", "/api/v1/files", "/api/v1/mail", "/api/v1/docs", "/api/v1/ops"): has_prefix = any(path == prefix or path.startswith(f"{prefix}/") for path in route_paths) self.assertEqual(prefix in expected_prefixes, has_prefix, f"{name}: {prefix}") @@ -238,12 +440,12 @@ class ModuleSystemTests(unittest.TestCase): def test_governance_template_routes_are_contributed_by_admin_module(self) -> None: access_only_app, _settings_obj = self._app_for_modules(()) - access_only_paths = {getattr(route, "path", "") for route in access_only_app.routes} + access_only_paths = _route_paths(access_only_app) self.assertIn("/api/v1/admin/users", access_only_paths) self.assertNotIn("/api/v1/admin/system/governance-templates", access_only_paths) admin_app, _settings_obj = self._app_for_modules(("admin",)) - admin_paths = {getattr(route, "path", "") for route in admin_app.routes} + admin_paths = _route_paths(admin_app) self.assertIn("/api/v1/admin/system/governance-templates", admin_paths) @@ -277,21 +479,25 @@ sys.meta_path.insert(0, Blocker()) from govoplan_core.server.app import create_app from govoplan_core.server.config import GovoplanServerConfig from govoplan_core.server.registry import build_platform_registry +from govoplan_core.db.session import reset_database -registry = build_platform_registry({enabled_modules!r}) -config = GovoplanServerConfig( - title="absence probe", - version="test", - enabled_modules={enabled_modules!r}, - base_routers=(), - post_module_routers=(), - extra_routers=(), - lifespan=None, - app_configurators=(), -) -app = create_app(config) -route_paths = [getattr(route, "path", "") for route in app.routes] -print(json.dumps({{"modules": [item.id for item in registry.manifests()], "routes": route_paths}}, sort_keys=True)) +try: + registry = build_platform_registry({enabled_modules!r}) + config = GovoplanServerConfig( + title="absence probe", + version="test", + enabled_modules={enabled_modules!r}, + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + app = create_app(config) + route_paths = [getattr(route, "path", "") for route in app.routes] + print(json.dumps({{"modules": [item.id for item in registry.manifests()], "routes": route_paths}}, sort_keys=True)) +finally: + reset_database(dispose=True) """ result = subprocess.run( [sys.executable, "-c", script], @@ -747,6 +953,82 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route record = read_module_installer_run(runtime_dir=root / "installer", run_id=result.run_id) self.assertEqual(result.run_id, record["run_id"]) + def test_module_installer_records_verified_local_artifact_integrity(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-artifact-integrity-", dir=_TEST_ROOT)) + artifact_path = root / "govoplan-files-0.1.4-py3-none-any.whl" + artifact_path.write_bytes(b"package artifact") + digest = hashlib.sha256(artifact_path.read_bytes()).hexdigest() + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + "artifact_integrity": { + "python": { + "ref": "govoplan-files==0.1.4", + "path": str(artifact_path), + "sha256": digest, + "sbom_url": "https://govoplan.example/sbom/files-0.1.4.spdx.json", + "provenance_url": "https://govoplan.example/provenance/files-0.1.4.intoto.jsonl", + }, + }, + }]) + session.commit() + + preflight = module_install_preflight( + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + runtime_dir=root / "installer", + ) + self.assertTrue(preflight.allowed) + self.assertEqual(digest, preflight.as_dict()["artifact_integrity"][0]["actual_sha256"]) + + result = run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=settings.database_url, + runtime_dir=root / "installer", + dry_run=True, + ) + + record = read_module_installer_run(runtime_dir=root / "installer", run_id=result.run_id) + verified = record["preflight"]["artifact_integrity"][0] + self.assertTrue(verified["verified"]) + self.assertEqual(str(artifact_path), verified["artifact_path"]) + + def test_module_installer_blocks_missing_integrity_in_production_mode(self) -> None: + plan = ModuleInstallPlan(items=(ModuleInstallPlanItem( + module_id="files", + action="install", + python_package="govoplan-files", + python_ref="govoplan-files==0.1.4", + ),)) + + with patch.dict(os.environ, {"GOVOPLAN_MODULE_INSTALLER_REQUIRE_ARTIFACT_INTEGRITY": "true"}): + preflight = module_install_preflight( + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + ) + + self.assertFalse(preflight.allowed) + self.assertIn("artifact_integrity_required", {issue.code for issue in preflight.issues}) + def test_supervised_module_install_rollback_restores_desired_modules(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-installer-desired-rollback-", dir=_TEST_ROOT)) settings = _settings(root) @@ -798,7 +1080,10 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route backup_script = ( "import json, os, pathlib; " "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); " - "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')" + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({" + "'snapshot': 'external', " + "'pgtools_url': os.environ.get('GOVOPLAN_DATABASE_URL_PGTOOLS')" + "}), encoding='utf-8')" ) backup_command = f"{sys.executable} -c {shlex.quote(backup_script)}" @@ -818,7 +1103,7 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route available=available_module_manifests(), current_enabled=("tenancy", "access"), desired_enabled=("tenancy", "access"), - database_url="postgresql://db.example.invalid/govoplan", + database_url="postgresql+psycopg://db.example.invalid/govoplan", runtime_dir=root / "installer", migrate_database=True, database_backup_command=backup_command, @@ -829,7 +1114,8 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route record = json.loads(result.record_path.read_text(encoding="utf-8")) database_backup = record["snapshot"]["database_backup"] self.assertEqual("external", database_backup["type"]) - self.assertEqual({"snapshot": "external"}, database_backup["metadata"]) + self.assertEqual("external", database_backup["metadata"]["snapshot"]) + self.assertEqual("postgresql://db.example.invalid/govoplan", database_backup["metadata"]["pgtools_url"]) self.assertEqual("backup", (result.record_path.parent / "database.external.backup").read_text(encoding="utf-8")) def test_module_installer_external_database_restore_check_runs_before_migrations(self) -> None: @@ -916,20 +1202,23 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route root = Path(tempfile.mkdtemp(prefix="govoplan-installer-request-queue-", dir=_TEST_ROOT)) runtime_dir = root / "installer" - queued = queue_module_installer_request( - runtime_dir=runtime_dir, - requested_by="user-1", - options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]}, - ) + with event_context(correlation_id="installer-trace-1"): + queued = queue_module_installer_request( + runtime_dir=runtime_dir, + requested_by="user-1", + options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]}, + ) request_id = str(queued["request_id"]) self.assertEqual("queued", queued["status"]) + self.assertEqual("installer-trace-1", queued["trace"]["correlation_id"]) self.assertEqual((request_id,), tuple(item["request_id"] for item in list_module_installer_requests(runtime_dir=runtime_dir))) claimed = claim_next_module_installer_request(runtime_dir=runtime_dir) self.assertIsNotNone(claimed) assert claimed is not None self.assertEqual(request_id, claimed["request_id"]) self.assertEqual("running", claimed["status"]) + self.assertEqual("installer-trace-1", claimed["trace"]["correlation_id"]) updated = update_module_installer_request( runtime_dir=runtime_dir, request_id=request_id, @@ -955,6 +1244,42 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertEqual("queued", retry["status"]) self.assertEqual(cancellable["request_id"], retry["retry_of"]) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + result = run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=settings.database_url, + runtime_dir=runtime_dir, + dry_run=True, + request_context={ + "request_id": request_id, + "requested_by": "user-1", + "trace": queued["trace"], + }, + ) + + record = json.loads(result.record_path.read_text(encoding="utf-8")) + self.assertEqual(request_id, record["request"]["request_id"]) + self.assertEqual("installer-trace-1", record["request"]["trace"]["correlation_id"]) + run_summary = list_module_installer_runs(runtime_dir=runtime_dir)[0] + self.assertEqual(request_id, run_summary["request_id"]) + self.assertEqual("installer-trace-1", run_summary["trace"]["correlation_id"]) + def test_module_installer_daemon_status_reports_heartbeat(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-installer-daemon-status-", dir=_TEST_ROOT)) runtime_dir = root / "installer" @@ -980,6 +1305,14 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route "version": "0.1.4", "python_package": "govoplan-files", "python_ref": "govoplan-files==0.1.4", + "artifact_integrity": { + "python": { + "ref": "govoplan-files==0.1.4", + "sha256": "0" * 64, + "sbom_url": "https://govoplan.example/sbom/files-0.1.4.spdx.json", + "provenance_url": "https://govoplan.example/provenance/files-0.1.4.intoto.jsonl", + } + }, "tags": ["official"], }], }), encoding="utf-8") @@ -989,6 +1322,7 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertEqual("files", catalog[0]["module_id"]) self.assertEqual("install", catalog[0]["action"]) self.assertEqual(["official"], catalog[0]["tags"]) + self.assertEqual("0" * 64, catalog[0]["artifact_integrity"]["python"]["sha256"]) validation = validate_module_package_catalog(catalog_path) self.assertTrue(validation["valid"]) @@ -1046,6 +1380,76 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertTrue(validation["trusted"]) self.assertEqual("stable", validation["channel"]) + def test_module_package_catalog_validates_remote_url_and_cache_fallback(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-remote-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + private_key_path = root / "catalog-private.pem" + signed_path = root / "catalog.signed.json" + cache_path = root / "catalog-cache.json" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + catalog_path.write_text(json.dumps({ + "channel": "stable", + "sequence": 42, + "generated_at": "2026-07-07T00:00:00Z", + "expires_at": "2030-12-31T23:59:59Z", + "modules": [{"module_id": "files"}], + }), encoding="utf-8") + sign_module_package_catalog( + path=catalog_path, + key_id="release-remote", + private_key_path=private_key_path, + output_path=signed_path, + ) + body = signed_path.read_text(encoding="utf-8") + + class _Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self) -> bytes: + return body.encode("utf-8") + + env = { + "GOVOPLAN_MODULE_PACKAGE_CATALOG_URL": "https://catalog.example.invalid/stable.json", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE": str(cache_path), + "GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE": "true", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS": "stable", + } + trusted_keys = {"release-remote": base64.b64encode(public_key).decode("ascii")} + with patch.dict(os.environ, env): + with patch("govoplan_core.core.module_package_catalog.urllib.request.urlopen", return_value=_Response()): + fetched = validate_module_package_catalog(trusted_keys=trusted_keys) + with patch( + "govoplan_core.core.module_package_catalog.urllib.request.urlopen", + side_effect=urllib.error.URLError("offline"), + ): + cached = validate_module_package_catalog(trusted_keys=trusted_keys) + + self.assertTrue(fetched["valid"], fetched["error"]) + self.assertEqual("url", fetched["source_type"]) + self.assertFalse(fetched["cache_used"]) + self.assertEqual("stable", fetched["channel"]) + self.assertEqual(42, fetched["sequence"]) + self.assertTrue(fetched["trusted"]) + self.assertEqual("release-remote", fetched["key_id"]) + self.assertTrue(cache_path.exists()) + self.assertTrue(cached["valid"], cached["error"]) + self.assertTrue(cached["cache_used"]) + self.assertEqual(str(cache_path), cached["cache_path"]) + self.assertTrue(cached["trusted"]) + def test_module_package_catalog_rejects_unapproved_channel_when_required(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-channel-", dir=_TEST_ROOT)) catalog_path = root / "catalog.json" @@ -1072,9 +1476,28 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertFalse(validation["valid"]) self.assertIn("expired", str(validation["error"])) + def test_module_package_catalog_rejects_not_before_catalog(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-not-before-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + catalog_path.write_text(json.dumps({ + "channel": "stable", + "sequence": 1, + "generated_at": "2026-07-07T00:00:00Z", + "not_before": "2030-01-01T00:00:00Z", + "expires_at": "2030-12-31T23:59:59Z", + "modules": [{"module_id": "files"}], + }), encoding="utf-8") + + validation = validate_module_package_catalog(catalog_path) + + self.assertFalse(validation["valid"]) + self.assertEqual("2030-01-01T00:00:00Z", validation["not_before"]) + self.assertIn("not valid before", str(validation["error"])) + def test_module_package_catalog_records_sequence_acceptance(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-sequence-", dir=_TEST_ROOT)) catalog_path = root / "catalog.json" + older_catalog_path = root / "catalog-older.json" state_path = root / "sequence-state.json" catalog_path.write_text(json.dumps({ "channel": "stable", @@ -1083,6 +1506,13 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route "expires_at": "2030-12-31T23:59:59Z", "modules": [{"module_id": "files"}], }), encoding="utf-8") + older_catalog_path.write_text(json.dumps({ + "channel": "stable", + "sequence": 6, + "generated_at": "2026-07-06T00:00:00Z", + "expires_at": "2030-12-31T23:59:59Z", + "modules": [{"module_id": "files"}], + }), encoding="utf-8") with patch.dict(os.environ, { "GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE": str(state_path), @@ -1094,9 +1524,163 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertTrue(validation["valid"], validation["error"]) record_module_package_catalog_acceptance(validation) replay = validate_module_package_catalog(catalog_path) + older = validate_module_package_catalog(older_catalog_path) + with patch.dict(os.environ, { + "GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE": str(state_path), + "GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE": "", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS": "", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE": "", + }): + same_non_strict = validate_module_package_catalog(catalog_path) self.assertFalse(replay["valid"]) self.assertIn("already accepted", str(replay["error"])) + self.assertFalse(older["valid"]) + self.assertIn("older than accepted", str(older["error"])) + self.assertTrue(same_non_strict["valid"], same_non_strict["error"]) + + def test_configuration_package_catalog_validates_signed_entries(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-configuration-package-catalog-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + private_key_path = root / "configuration-private.pem" + signed_path = root / "catalog.signed.json" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + catalog_path.write_text(json.dumps({ + "channel": "stable", + "sequence": 3, + "generated_at": "2026-07-07T00:00:00Z", + "expires_at": "2030-12-31T23:59:59Z", + "packages": [ + { + "package_id": "govoplan.application-handling.basic", + "name": "Basic application handling", + "version": "0.1.0", + "publisher": "ADD ideas", + "category": "workflow", + "artifact_ref": "git+ssh://git.example.invalid/config-packages.git@application-basic-v0.1.0", + "artifact_sha256": "a" * 64, + "required_modules": [{"module_id": "access", "version": ">=0.1.0"}], + "required_capabilities": [CONFIGURATION_PROVIDER_CAPABILITY, "access.configuration"], + "fragments": [ + { + "module_id": "access", + "fragment_type": "access_roles", + "fragment_id": "clerks", + "payload": {"access_roles": [{"slug": "application-clerk"}]}, + } + ], + "data_requirements": [{"key": "service_name", "label": "Service name"}], + "tags": ["official", "example"], + } + ], + }), encoding="utf-8") + + unsigned = validate_configuration_package_catalog(catalog_path) + self.assertTrue(unsigned["valid"], unsigned["error"]) + self.assertFalse(unsigned["signed"]) + self.assertEqual("govoplan.application-handling.basic", unsigned["packages"][0]["package_id"]) + self.assertEqual("access", unsigned["packages"][0]["required_modules"][0]["module_id"]) + self.assertIn("unsigned", " ".join(unsigned["warnings"]).lower()) + + sign_configuration_package_catalog( + path=catalog_path, + key_id="configuration-release-1", + private_key_path=private_key_path, + output_path=signed_path, + ) + validation = validate_configuration_package_catalog( + signed_path, + require_trusted=True, + approved_channels=("stable",), + trusted_keys={"configuration-release-1": base64.b64encode(public_key).decode("ascii")}, + ) + self.assertTrue(validation["valid"], validation["error"]) + self.assertTrue(validation["signed"]) + self.assertTrue(validation["trusted"]) + self.assertEqual("stable", validation["channel"]) + + catalog = configuration_package_catalog( + signed_path, + require_trusted=True, + approved_channels=("stable",), + trusted_keys={"configuration-release-1": base64.b64encode(public_key).decode("ascii")}, + ) + self.assertEqual(("official", "example"), tuple(catalog[0]["tags"])) + + def test_configuration_provider_contract_is_runtime_checkable(self) -> None: + class AccessConfigurationProvider: + module_id = "access" + + def describe(self) -> ConfigurationProviderDescription: + return ConfigurationProviderDescription(module_id=self.module_id, fragment_types=("access_roles",), exported_scopes=("tenant",)) + + def preflight(self, fragment: ConfigurationPackageFragment, context: ConfigurationPreflightContext) -> ConfigurationPreflightResult: + del context + return ConfigurationPreflightResult( + required_data=(ConfigurationRequiredData(key="service_name", label="Service name"),), + plan=(ConfigurationPlanItem(action="create", module_id=fragment.module_id, fragment_type=fragment.fragment_type, fragment_id=fragment.fragment_id),), + ) + + def apply(self, fragment: ConfigurationPackageFragment, supplied_data: dict[str, object], context: ConfigurationPreflightContext) -> ConfigurationApplyResult: + del supplied_data, context + return ConfigurationApplyResult(created_refs={fragment.fragment_id or fragment.fragment_type: "role:application-clerk"}) + + def export(self, selection: ConfigurationExportSelection, context: ConfigurationPreflightContext) -> ConfigurationExportResult: + del selection, context + return ConfigurationExportResult(fragments=(ConfigurationPackageFragment(module_id=self.module_id, fragment_type="access_roles", fragment_id="clerks"),)) + + def health(self, import_result: ConfigurationApplyResult, context: ConfigurationPreflightContext): + del import_result, context + return () + + provider = AccessConfigurationProvider() + fragment = ConfigurationPackageFragment(module_id="access", fragment_type="access_roles", fragment_id="clerks") + + self.assertIsInstance(provider, ConfigurationProvider) + self.assertEqual(("access_roles",), provider.describe().fragment_types) + self.assertEqual("service_name", provider.preflight(fragment, ConfigurationPreflightContext()).required_data[0].key) + self.assertEqual("role:application-clerk", provider.apply(fragment, {}, ConfigurationPreflightContext()).created_refs["clerks"]) + + package = { + "package_id": "govoplan.test", + "name": "Test package", + "version": "1.0.0", + "required_modules": [{"module_id": "access", "version": "1.0.0"}], + "required_capabilities": [CONFIGURATION_PROVIDER_CAPABILITY, "access.configuration"], + "data_requirements": [{"key": "service_name", "label": "Service name"}], + "fragments": [{"module_id": "access", "fragment_type": "access_roles", "fragment_id": "clerks", "payload": {"access_roles": []}}], + } + missing_context = ConfigurationPreflightContext( + installed_modules={"access": "1.0.0"}, + capabilities=frozenset({CONFIGURATION_PROVIDER_CAPABILITY, "access.configuration"}), + ) + missing = dry_run_configuration_package(package, [provider], missing_context) + self.assertIn("required_data_missing", {item.code for item in missing.diagnostics}) + self.assertEqual("service_name", missing.required_data[0].key) + + ready_context = ConfigurationPreflightContext( + supplied_data={"service_name": "Application handling"}, + installed_modules={"access": "1.0.0"}, + capabilities=frozenset({CONFIGURATION_PROVIDER_CAPABILITY, "access.configuration"}), + ) + dry_run = dry_run_configuration_package(package, [provider], ready_context) + self.assertFalse([item for item in dry_run.diagnostics if item.severity == "blocker"]) + self.assertEqual("create", dry_run.plan[0].action) + + applied = apply_configuration_package(package, [provider], ready_context) + self.assertEqual("role:application-clerk", applied.created_refs["clerks"]) + + exported = export_configuration_package([provider], ConfigurationExportSelection(module_ids=("access",)), ready_context) + self.assertEqual("access_roles", exported.fragments[0].fragment_type) def test_module_license_decision_is_observe_only_unless_enforced(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-license-observe-", dir=_TEST_ROOT)) @@ -1149,6 +1733,191 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertTrue(validation["trusted"]) self.assertEqual(["module.mail"], validation["features"]) + def test_module_license_issuer_writes_signed_license(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-license-issuer-", dir=_TEST_ROOT)) + private_key_path = root / "license-private.pem" + license_path = root / "license.json" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + + issue_module_license( + path=license_path, + license_id="license-2026", + subject="GovOPlaN Test", + features=("module.mail", "support.standard"), + valid_from="2026-01-01T00:00:00Z", + valid_until="2030-12-31T23:59:59Z", + signing_key_id="license-key-1", + signing_private_key_path=private_key_path, + issuer="GovOPlaN Release", + ) + trusted_keys = {"license-key-1": base64.b64encode(public_key).decode("ascii")} + + validation = validate_module_license(license_path, trusted_keys=trusted_keys, require_trusted=True) + diagnostics = module_license_diagnostics( + license_path, + trusted_keys=trusted_keys, + require_trusted=True, + required_features=("module.mail",), + ) + + self.assertTrue(validation["valid"], validation["error"]) + self.assertEqual("license-2026", validation["license_id"]) + self.assertTrue(diagnostics["trusted"]) + self.assertEqual([], diagnostics["missing_features"]) + self.assertIn("module.mail", diagnostics["features"]) + + def test_module_license_diagnostics_report_expired_unsigned_missing_and_missing_feature(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-license-diagnostics-", dir=_TEST_ROOT)) + missing_path = root / "missing.json" + expired_path = root / "expired.json" + unsigned_path = root / "unsigned.json" + feature_path = root / "feature.json" + private_key_path = root / "license-private.pem" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + trusted_keys = {"license-key-1": base64.b64encode(public_key).decode("ascii")} + expired_path.write_text(json.dumps({ + "license_id": "expired", + "subject": "Expired", + "features": ["module.mail"], + "valid_from": "2020-01-01T00:00:00Z", + "valid_until": "2020-01-02T00:00:00Z", + }), encoding="utf-8") + unsigned_path.write_text(json.dumps({ + "license_id": "unsigned", + "subject": "Unsigned", + "features": ["module.mail"], + "valid_from": "2026-01-01T00:00:00Z", + "valid_until": "2030-12-31T23:59:59Z", + }), encoding="utf-8") + issue_module_license( + path=feature_path, + license_id="feature", + subject="Feature", + features=("module.files",), + valid_from="2026-01-01T00:00:00Z", + valid_until="2030-12-31T23:59:59Z", + signing_key_id="license-key-1", + signing_private_key_path=private_key_path, + ) + + with patch.dict(os.environ, {"GOVOPLAN_LICENSE_ENFORCEMENT": "true"}): + missing = module_license_diagnostics(missing_path, required_features=("module.mail",), require_trusted=True) + expired = module_license_diagnostics(expired_path, required_features=("module.mail",), require_trusted=False) + unsigned = module_license_diagnostics(unsigned_path, required_features=("module.mail",), require_trusted=True) + missing_feature = module_license_diagnostics( + feature_path, + trusted_keys=trusted_keys, + require_trusted=True, + required_features=("module.mail",), + ) + + self.assertFalse(missing["valid"]) + self.assertFalse(missing["allowed"]) + self.assertFalse(expired["valid"]) + self.assertIn("expired", str(expired["reason"])) + self.assertFalse(unsigned["valid"]) + self.assertFalse(unsigned["signed"]) + self.assertFalse(unsigned["trusted"]) + self.assertTrue(missing_feature["valid"], missing_feature["reason"]) + self.assertTrue(missing_feature["trusted"]) + self.assertEqual(["module.mail"], missing_feature["missing_features"]) + self.assertFalse(missing_feature["allowed"]) + + def test_module_installer_cli_issues_and_validates_license(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-license-cli-", dir=_TEST_ROOT)) + private_key_path = root / "license-private.pem" + license_path = root / "license.json" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + trusted_key = base64.b64encode(public_key).decode("ascii") + + issue_result = subprocess.run( + [ + sys.executable, + "-m", + "govoplan_core.commands.module_installer", + "--issue-license", + str(license_path), + "--license-id", + "cli-license", + "--license-subject", + "CLI Test", + "--license-feature", + "module.mail", + "--license-valid-from", + "2026-01-01T00:00:00Z", + "--license-valid-until", + "2030-12-31T23:59:59Z", + "--license-signing-key-id", + "license-key-1", + "--license-signing-private-key", + str(private_key_path), + "--format", + "json", + ], + cwd=str(Path(__file__).resolve().parents[1]), + env=os.environ.copy(), + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(0, issue_result.returncode, issue_result.stderr) + self.assertTrue(json.loads(issue_result.stdout)["issued"]) + + validate_result = subprocess.run( + [ + sys.executable, + "-m", + "govoplan_core.commands.module_installer", + "--validate-license", + str(license_path), + "--license-trusted-key", + f"license-key-1={trusted_key}", + "--require-trusted-license", + "--license-required-feature", + "module.mail", + "--format", + "json", + ], + cwd=str(Path(__file__).resolve().parents[1]), + env=os.environ.copy(), + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(0, validate_result.returncode, validate_result.stderr) + diagnostics = json.loads(validate_result.stdout) + self.assertTrue(diagnostics["valid"], diagnostics["reason"]) + self.assertTrue(diagnostics["trusted"]) + self.assertEqual([], diagnostics["missing_features"]) + def test_module_installer_validates_package_catalog_from_cli(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-cli-", dir=_TEST_ROOT)) catalog_path = root / "catalog.json" @@ -1340,14 +2109,14 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route response = client.get("/api/v1/platform/modules") self.assertEqual(response.status_code, 200, response.text) self.assertEqual({"tenancy", "access"}, {item["id"] for item in response.json()["modules"]}) - self.assertFalse(any(getattr(route, "path", "") == "/api/v1/files" for route in app.routes)) + self.assertFalse("/api/v1/files" in _route_paths(app)) result = lifecycle.apply_enabled_modules(("files",), migrate=False) self.assertEqual(("files",), result.activated_modules) response = client.get("/api/v1/platform/modules") self.assertEqual(response.status_code, 200, response.text) self.assertEqual({"tenancy", "access", "files"}, {item["id"] for item in response.json()["modules"]}) - self.assertTrue(any(getattr(route, "path", "") == "/api/v1/files" for route in app.routes)) + self.assertTrue("/api/v1/files" in _route_paths(app)) self.assertEqual(401, client.get("/api/v1/files").status_code) result = lifecycle.apply_enabled_modules((), migrate=False) @@ -1386,15 +2155,19 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route def test_module_migrations_are_registered_only_for_enabled_modules(self) -> None: core_plan = migration_metadata_plan(build_platform_registry(())) - self.assertEqual((), core_plan.script_locations) + self.assertEqual(1, len(core_plan.script_locations)) + self.assertTrue(core_plan.script_locations[0].endswith("govoplan_access/backend/migrations/versions")) self.assertEqual(1, len(core_plan.metadata)) files_plan = migration_metadata_plan(build_platform_registry(("files",))) - self.assertEqual(1, len(files_plan.script_locations)) - self.assertTrue(files_plan.script_locations[0].endswith("govoplan_files/backend/migrations/versions")) + self.assertEqual(2, len(files_plan.script_locations)) + files_locations = "\n".join(files_plan.script_locations) + self.assertIn("govoplan_access/backend/migrations/versions", files_locations) + self.assertIn("govoplan_files/backend/migrations/versions", files_locations) full_plan = migration_metadata_plan(build_platform_registry(("campaigns", "files", "mail"))) locations = "\n".join(full_plan.script_locations) + self.assertIn("govoplan_access/backend/migrations/versions", locations) self.assertIn("govoplan_campaign/backend/migrations/versions", locations) self.assertIn("govoplan_files/backend/migrations/versions", locations) self.assertIn("govoplan_mail/backend/migrations/versions", locations) diff --git a/tests/test_pagination.py b/tests/test_pagination.py new file mode 100644 index 0000000..922f7d3 --- /dev/null +++ b/tests/test_pagination.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import unittest + +from govoplan_core.core.pagination import ( + KeysetCursorError, + decode_keyset_cursor, + encode_keyset_cursor, + keyset_query_fingerprint, +) + + +class PaginationCursorTests(unittest.TestCase): + def test_keyset_cursor_round_trips_and_rejects_query_mismatch(self) -> None: + fingerprint = keyset_query_fingerprint("audit.admin", {"scope": "system", "page_size": 25}) + cursor = encode_keyset_cursor( + "audit.admin", + fingerprint=fingerprint, + values={"id": "row-1", "sort_value": "2026-01-01T00:00:00+00:00"}, + ) + + self.assertEqual( + decode_keyset_cursor("audit.admin", cursor, fingerprint=fingerprint), + {"id": "row-1", "sort_value": "2026-01-01T00:00:00+00:00"}, + ) + with self.assertRaises(KeysetCursorError): + decode_keyset_cursor("audit.admin", cursor, fingerprint=keyset_query_fingerprint("audit.admin", {"scope": "tenant"})) + with self.assertRaises(KeysetCursorError): + decode_keyset_cursor("files.list", cursor, fingerprint=fingerprint) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_policy_contracts.py b/tests/test_policy_contracts.py new file mode 100644 index 0000000..e8b29fa --- /dev/null +++ b/tests/test_policy_contracts.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import unittest + +from govoplan_core.core.configuration_safety import ( + classify_configuration_field, + configuration_safety_catalog, + high_impact_configuration_fields, + plan_configuration_change, + ui_managed_configuration_fields_requiring_approval, +) +from govoplan_core.core.policy import ( + PolicyDecision, + PolicySourceStep, + parse_policy_source_path, + policy_source_path, + policy_source_step, +) + + +class PolicyContractTests(unittest.TestCase): + def test_policy_source_paths_are_stable_and_round_trip(self) -> None: + self.assertEqual(policy_source_path("system"), "system") + self.assertEqual(policy_source_path("tenant", "tenant-1"), "tenant:tenant-1") + self.assertEqual(policy_source_path("campaign", "campaign/with space"), "campaign:campaign%2Fwith%20space") + + tenant_ref = parse_policy_source_path("tenant:tenant-1") + self.assertEqual(tenant_ref.scope_type, "tenant") + self.assertEqual(tenant_ref.scope_id, "tenant-1") + self.assertEqual(tenant_ref.path, "tenant:tenant-1") + + campaign_ref = parse_policy_source_path("campaign:campaign%2Fwith%20space") + self.assertEqual(campaign_ref.scope_id, "campaign/with space") + + def test_policy_source_step_and_decision_serialize_for_api_responses(self) -> None: + source = policy_source_step( + "tenant", + "Tenant", + "tenant-1", + applied_fields=("allow_lower_level_limits",), + policy={"allow_lower_level_limits": {"audit_detail_level": False}}, + ) + self.assertEqual(source["path"], "tenant:tenant-1") + self.assertEqual(source["applied_fields"], ["allow_lower_level_limits"]) + + decision = PolicyDecision( + allowed=False, + reason="Parent policy locks lower-level changes.", + source_path=(PolicySourceStep.from_mapping(source),), + requirements=("audit_detail_level",), + details={"blocked_fields": ["audit_detail_level"]}, + ) + payload = decision.to_dict() + self.assertFalse(payload["allowed"]) + self.assertEqual(payload["source_path"][0]["path"], "tenant:tenant-1") + self.assertEqual(payload["requirements"], ["audit_detail_level"]) + + def test_configuration_safety_catalog_classifies_ui_and_env_managed_fields(self) -> None: + catalog = {item.key: item for item in configuration_safety_catalog()} + + module_plan = catalog["module_management.install_plan"] + self.assertTrue(module_plan.ui_managed) + self.assertEqual(module_plan.risk, "destructive") + self.assertTrue(module_plan.dry_run_required) + self.assertTrue(module_plan.maintenance_required) + self.assertTrue(module_plan.two_person_approval_required) + self.assertTrue(module_plan.rollback_history_required) + + connector_profiles = catalog["files.connector_profiles"] + self.assertEqual(connector_profiles.secret_handling, "reference_only") + self.assertTrue(connector_profiles.policy_explanation_required) + self.assertIn("files:file:admin", connector_profiles.required_scopes) + + database_url = catalog["DATABASE_URL"] + self.assertFalse(database_url.ui_managed) + self.assertEqual(database_url.secret_handling, "env_only") + self.assertEqual(database_url.storage, "environment") + + self.assertIs(classify_configuration_field("MASTER_KEY_B64"), catalog["MASTER_KEY_B64"]) + self.assertIsNone(classify_configuration_field("missing.setting")) + self.assertNotIn("DATABASE_URL", {item.key for item in configuration_safety_catalog(include_env_only=False)}) + + def test_configuration_safety_helpers_surface_guardrail_sets(self) -> None: + high_impact = {item.key for item in high_impact_configuration_fields()} + approval = {item.key for item in ui_managed_configuration_fields_requiring_approval()} + + self.assertIn("module_management.install_plan", high_impact) + self.assertIn("privacy_retention_policy", approval) + self.assertIn("files.connector_profiles", approval) + self.assertNotIn("DATABASE_URL", approval) + + def test_configuration_change_safety_plan_enforces_guardrails(self) -> None: + blocked = plan_configuration_change( + "files.connector_profiles", + actor_scopes=("tenant:*",), + value={"password": "plain-secret"}, + dry_run=False, + maintenance_mode=False, + approval_count=1, + ) + self.assertFalse(blocked.allowed) + self.assertIn("dry_run_required", blocked.blockers) + self.assertIn("two_person_approval_required", blocked.blockers) + self.assertIn("secret_reference_required", blocked.blockers) + self.assertEqual(blocked.audit_event, "files.connector_profile.updated") + self.assertTrue(blocked.rollback_history_required) + + allowed = plan_configuration_change( + "files.connector_profiles", + actor_scopes=("tenant:*",), + value={"secret_ref": "vault://files/smb"}, + dry_run=True, + approval_count=2, + ) + self.assertTrue(allowed.allowed, allowed.to_dict()) + self.assertEqual(allowed.secret_handling, "reference_only") + + env_only = plan_configuration_change("DATABASE_URL", actor_scopes=("system:*",), value="postgresql://example") + self.assertFalse(env_only.allowed) + self.assertIn("deployment_managed", env_only.blockers) + self.assertIn("env_only_secret", env_only.blockers) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_migration_audit.py b/tests/test_release_migration_audit.py new file mode 100644 index 0000000..1cba705 --- /dev/null +++ b/tests/test_release_migration_audit.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "release-migration-audit.py" + + +def load_audit_module(): + spec = importlib.util.spec_from_file_location("release_migration_audit", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {SCRIPT}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class ReleaseMigrationAuditTests(unittest.TestCase): + def test_typed_alembic_variables_are_parsed(self) -> None: + audit = load_audit_module() + with tempfile.TemporaryDirectory(prefix="migration-audit-test-") as directory: + path = Path(directory) / "1234_example.py" + path.write_text( + """ +from __future__ import annotations +from typing import Sequence, Union + +revision: str = "1234" +down_revision: Union[str, None] = "base" +branch_labels: Union[str, Sequence[str], None] = None +""", + encoding="utf-8", + ) + + migration = audit.parse_migration_file("govoplan-core", path) + + self.assertIsNotNone(migration) + self.assertEqual(migration.revision, "1234") + self.assertEqual(migration.down_revisions, ("base",)) + self.assertEqual(migration.branch_labels, ()) + + def test_release_baseline_matches_current_heads_in_strict_report(self) -> None: + audit = load_audit_module() + migrations = [ + audit.Migration("govoplan-core", Path("base.py"), "base", (), ()), + audit.Migration("govoplan-core", Path("head.py"), "head", ("base",), ()), + ] + report = audit.build_report( + migrations, + baseline={ + "version": 1, + "releases": [ + { + "release": "0.1.0", + "heads": [{"owner": "govoplan-core", "revision": "head"}], + } + ], + }, + baseline_file=Path("docs/migration-release-baselines.json"), + workspace_root=Path("/workspace"), + ) + + self.assertEqual(report["current_heads"], ["head"]) + self.assertEqual(report["graph_errors"], []) + self.assertEqual(report["strict_errors"], []) + + def test_owner_heads_are_accepted_for_subset_installs(self) -> None: + audit = load_audit_module() + migrations = [ + audit.Migration("govoplan-core", Path("base.py"), "base", (), ()), + audit.Migration("govoplan-core", Path("core_head.py"), "core-head", ("base",), ()), + ] + report = audit.build_report( + migrations, + baseline={ + "version": 1, + "releases": [ + { + "release": "0.1.0", + "heads": [{"owner": "govoplan-files", "revision": "files-head"}], + "owner_heads": [{"owner": "govoplan-core", "revisions": ["core-head"]}], + } + ], + }, + baseline_file=Path("docs/migration-release-baselines.json"), + workspace_root=Path("/workspace"), + ) + + self.assertEqual(report["current_heads"], ["core-head"]) + self.assertEqual(report["strict_errors"], []) + + def test_records_current_release_baseline(self) -> None: + audit = load_audit_module() + migrations = [ + audit.Migration("govoplan-core", Path("base.py"), "base", (), ()), + audit.Migration("govoplan-core", Path("core_head.py"), "core-head", ("base",), ()), + audit.Migration("govoplan-files", Path("files_head.py"), "files-head", ("core-head",), ()), + ] + report = audit.build_report( + migrations, + baseline={"version": 1, "releases": []}, + baseline_file=Path("docs/migration-release-baselines.json"), + workspace_root=Path("/workspace"), + ) + + baseline = audit.record_release_baseline( + {"version": 1, "releases": []}, + report, + release="0.2.0", + replace=False, + ) + + release = baseline["releases"][0] + self.assertEqual(release["release"], "0.2.0") + self.assertEqual(release["squash_policy"], "reviewed-manual") + self.assertEqual(release["heads"], [{"owner": "govoplan-files", "revision": "files-head"}]) + self.assertIn({"owner": "govoplan-core", "revisions": ["core-head"]}, release["owner_heads"]) + + def test_missing_down_revision_is_a_graph_error(self) -> None: + audit = load_audit_module() + migrations = [ + audit.Migration("govoplan-core", Path("head.py"), "head", ("missing",), ()), + ] + report = audit.build_report( + migrations, + baseline={"version": 1, "releases": []}, + baseline_file=Path("docs/migration-release-baselines.json"), + workspace_root=Path("/workspace"), + ) + + self.assertIn("references missing down_revision", report["graph_errors"][0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/index.html b/webui/index.html index e21c903..a48955b 100644 --- a/webui/index.html +++ b/webui/index.html @@ -3,6 +3,13 @@ + + + + + + + GovOPlaN diff --git a/webui/package-lock.json b/webui/package-lock.json index 0c35e35..3814b3a 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -12,14 +12,17 @@ "@govoplan/admin-webui": "file:../../govoplan-admin/webui", "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", + "@govoplan/dashboard-webui": "file:../../govoplan-dashboard/webui", + "@govoplan/docs-webui": "file:../../govoplan-docs/webui", "@govoplan/files-webui": "file:../../govoplan-files/webui", - "@govoplan/mail-webui": "file:../../govoplan-mail/webui" + "@govoplan/mail-webui": "file:../../govoplan-mail/webui", + "@govoplan/ops-webui": "file:../../govoplan-ops/webui" }, "devDependencies": { "@types/react": "^19.0.2", "@types/react-dom": "^19.0.2", "@vitejs/plugin-react": "^4.3.4", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1", @@ -28,7 +31,7 @@ "vite": "^6.0.6" }, "peerDependencies": { - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1" @@ -42,7 +45,7 @@ }, "peerDependencies": { "@govoplan/core-webui": "^0.1.6", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1" @@ -61,7 +64,7 @@ }, "peerDependencies": { "@govoplan/core-webui": "^0.1.6", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1" @@ -78,7 +81,7 @@ "peerDependencies": { "@govoplan/core-webui": "^0.1.6", "@vitejs/plugin-react": "^4.3.4", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1", @@ -102,7 +105,7 @@ }, "peerDependencies": { "@govoplan/core-webui": "^0.1.6", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1" @@ -113,13 +116,48 @@ } } }, + "../../govoplan-dashboard/webui": { + "name": "@govoplan/dashboard-webui", + "version": "0.1.6", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "lucide-react": "^1.23.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "../../govoplan-docs/webui": { + "name": "@govoplan/docs-webui", + "version": "0.1.6", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "@vitejs/plugin-react": "^4.3.4", + "lucide-react": "^1.23.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1", + "typescript": "^5.7.2", + "vite": "^6.0.6" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-files/webui": { "name": "@govoplan/files-webui", "version": "0.1.6", "peerDependencies": { "@govoplan/core-webui": "^0.1.6", "@vitejs/plugin-react": "^4.3.4", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1", @@ -140,7 +178,7 @@ }, "peerDependencies": { "@govoplan/core-webui": "^0.1.6", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1" @@ -151,6 +189,25 @@ } } }, + "../../govoplan-ops/webui": { + "name": "@govoplan/ops-webui", + "version": "0.1.6", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "@vitejs/plugin-react": "^4.3.4", + "lucide-react": "^1.23.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1", + "typescript": "^5.7.2", + "vite": "^6.0.6" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -891,6 +948,14 @@ "resolved": "../../govoplan-campaign/webui", "link": true }, + "node_modules/@govoplan/dashboard-webui": { + "resolved": "../../govoplan-dashboard/webui", + "link": true + }, + "node_modules/@govoplan/docs-webui": { + "resolved": "../../govoplan-docs/webui", + "link": true + }, "node_modules/@govoplan/files-webui": { "resolved": "../../govoplan-files/webui", "link": true @@ -899,6 +964,10 @@ "resolved": "../../govoplan-mail/webui", "link": true }, + "node_modules/@govoplan/ops-webui": { + "resolved": "../../govoplan-ops/webui", + "link": true + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -1722,9 +1791,9 @@ } }, "node_modules/lucide-react": { - "version": "0.555.0", - "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.555.0.tgz", - "integrity": "sha512-D8FvHUGbxWBRQM90NZeIyhAvkFfsh3u9ekrMvJ30Z6gnpBHS6HC6ldLg7tL45hwiIz/u66eKDtdA23gwwGsAHA==", + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.23.0.tgz", + "integrity": "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw==", "dev": true, "license": "ISC", "peerDependencies": { diff --git a/webui/package.json b/webui/package.json index 3381597..88d4ae3 100644 --- a/webui/package.json +++ b/webui/package.json @@ -20,7 +20,8 @@ "dev": "vite --host 127.0.0.1 --port 5173", "build": "tsc && vite build", "preview": "vite preview --host 127.0.0.1 --port 4173", - "test:module-capabilities": "rm -rf .module-test-build && mkdir -p .module-test-build && printf '{\"type\":\"commonjs\"}\n' > .module-test-build/package.json && tsc -p tsconfig.module-tests.json && node .module-test-build/tests/module-capabilities.test.js", + "audit:i18n-structural": "node scripts/audit-i18n-structural.mjs", + "test:module-capabilities": "rm -rf .module-test-build && mkdir -p .module-test-build && printf '{\"type\":\"commonjs\"}\n' > .module-test-build/package.json && tsc -p tsconfig.module-tests.json && node .module-test-build/tests/module-capabilities.test.js && node .module-test-build/tests/privacy-policy.test.js", "test:module-permutations": "node scripts/test-module-permutations.mjs", "test:mail-components": "rm -rf .component-test-build && mkdir -p .component-test-build && printf '{\"type\":\"commonjs\"}\\n' > .component-test-build/package.json && tsc -p tsconfig.component-tests.json && node .component-test-build/tests/mail-components.test.js" }, @@ -29,14 +30,17 @@ "@govoplan/admin-webui": "file:../../govoplan-admin/webui", "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", + "@govoplan/dashboard-webui": "file:../../govoplan-dashboard/webui", + "@govoplan/docs-webui": "file:../../govoplan-docs/webui", "@govoplan/files-webui": "file:../../govoplan-files/webui", - "@govoplan/mail-webui": "file:../../govoplan-mail/webui" + "@govoplan/mail-webui": "file:../../govoplan-mail/webui", + "@govoplan/ops-webui": "file:../../govoplan-ops/webui" }, "devDependencies": { "@types/react": "^19.0.2", "@types/react-dom": "^19.0.2", "@vitejs/plugin-react": "^4.3.4", - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1", @@ -45,7 +49,7 @@ "vite": "^6.0.6" }, "peerDependencies": { - "lucide-react": "^0.555.0", + "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^7.1.1" diff --git a/webui/package.release.json b/webui/package.release.json index ada5f24..c1739b7 100644 --- a/webui/package.release.json +++ b/webui/package.release.json @@ -25,9 +25,12 @@ "@govoplan/access-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git#v0.1.6", "@govoplan/admin-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git#v0.1.6", "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git#v0.1.6", + "@govoplan/dashboard-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-dashboard.git#v0.1.6", + "@govoplan/docs-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-docs.git#v0.1.6", "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.6", "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.6", - "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.6" + "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.6", + "@govoplan/ops-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-ops.git#v0.1.6" }, "devDependencies": { "lucide-react": "^0.555.0", diff --git a/webui/public/app-icon.svg b/webui/public/app-icon.svg new file mode 100644 index 0000000..d6ce24f --- /dev/null +++ b/webui/public/app-icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/webui/public/apple-touch-icon.png b/webui/public/apple-touch-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..5ec37db7b97167661c62eeb7d9e3828614e2d16b GIT binary patch literal 7866 zcmV;r9!24aP)q9rTv|NkIZvmSw zrsc`AHj;`IH!ZiNcSS)rH8uHCQaOg8Mqk_Vnq1>Gpa~`uI zU~??VKP>aRO9=*BZ;*<7=1RZIlO93{EQ*3+vEsKRnNO%JrX)$tV#_u(szkk^udC}4 zf7*K4hb*eXH$+~{DH0ND4yH6rs3eXfegvMU{cY6eAIpglfd!E^AN(%00jy;N>TEiEx9>&F_{{{3z3w znHeDj1qCiVE-$VEH?9IV4!a}maV{40YDh^KjjD9bCh3?>aeO98*Q`oP=BM{d5%28n zYJd2`pDv!Eg_Pnm23TY|3Y;#y?m|2sFCLc%i{ee?M#94Y@6_$ej;&xIB~;`6w(b#%|L-(I(v) zdNqtkRh|hYczhzk$(f|)<7W%16u0zuw?76K&CoBUO&MpE0oZMJN{UMHl>~4&oJO0I zHWf~Vchxv}SB-gx4@DQZ9(#yT-$k8#FCK9{FPWOX(#67 z6eV_r57#-lv)aLZXQMnY94D5_F<(TA*8vt047To-iaczzJz7#+R7!P44K@?a#+mY= zSHnb12!wCGTQn5Q~6M1q=A=U_rL{)mGDQ3kV_D zTwPDVS8lW^8TMu0AOtqzFny0*}qK1;9-0gCaYcuo( z-dthhuEqkR&3GA8JY|1rS$E1m-F^&1t8V9#ko8v^$a6x-qUTUf~8*PT}iP|ng&i^eWvDx4 zPQe#jT!!Xb8}^jT{S`U8qj4o-YVP{b7juK(%AeL6=T93VXm^2|s)~(OPd%FBF2}`u ztf7ErgVMP&{*)C(VRQAy@S86tCXHV(WeTf+|K8#<y;}dbw5~Nd zma6CH{x)4S7GZXFn(4V2;_(Q{R0?@fKfw!?p?`MKHHv^@o1n&J;kAAnH~6j8xpVs; z{(Ebk{A4IjT*rUSqOqw(x6uQ>a;ht9^%~w#BGE<0C&!qbok3Monv7Xs5!_T}<9#&_ z8ojwG?k{&Q^3ZTxxAAoov=D-p;Le9>Wai=gdxM@E}k>VcG!;qgH_``U@dqDGs;a75+3&(HC-o=Dd3kX%z@ zrNXJtT&l~crN!Buy`4xV=DLo>ntY!P%}Nf{O;JGYz`7U${i??6?}FO-F( z+P2>$pl{}(QA$;&@V#G(wJJjmSW05(Oz zj0O0ku?Ss#?O7$}i?Jzi-`+y%J?4xT8MWCND=(&|)la4W4^@`NrlP$4vj@@gv1@wt zT^y*oM9cs-hm*QJuLocuJi*Y3XNk{*uv;@1BfAny3&4)76*E9Wg%7?k$HRYANU2G> zSIt^1oc0$|5)4pZ>*Y#wai!16)+!HuBlDWwS5rY()UMpa(_LCl(@lQ@z~oR56P-tp z(Rr*EW7@n(Dd{`Yoi%RJN-U%#cO9GO2m8E+oh&ARE{_N5e6+TdvU?RBPxdsIm{-tc zn^)hI)?X9EUlT-ClZwS@o7EhEwpWVq^ zO$otPKW*Ohb}#lem+-)^hBUKBGeHYQAyCth_H~M4rDpqe)NH?w*upGBr=DYRVhFp= zA*a#kBB7~qUGHH82L@y8ud=gM=g7S=#mY!oF*Q2^G?lHrIQ{OX5-f@!rO|1TW`ZuQ zsKxGB-^}1}7t(O;n*o@e9A)I>b0lY{u<77$pC0Vfd`vSSfzO>>;3wA=Y99MBrzrFk zd2u)RXlX2=T7w&S3#dX3jI|S-h4X%V?D>2>_3T;h+)xS z<_?9=5sSyxJ&sv9yP)##BXRzC)A|oD5<;QJ<3`k%vaP9ZqaI} zS5O&jqcYe=DiLS6{ds1Fd$GZqns7=f8J`%|-bj;iPk)5BSFJJhRKRJcletZ$Y;EFW;U5fKFVMC(kGR<^C z^5}S+{gtZ}qu0rt&rP_*gRiEDMsMb}Rkzi8G#g73K`Rz3o(#utI15Xu-}goU;`7r? z4RkU)(vM6gR=jR>1I*4B_?OesWd$t?Y!L8}EccQuFT_>kpcp95tcd_rm$|T71W65| z{xuP_w>W?h#+_2xJ;hXRzn;qN*Re1)&g^hMb7N;vQAe4t&O1F+U+Ci#c=N_ zqfIc3r;mq>HsPw27HhLK)kM&?EU3soKYh+<6AWW$!u&2rVSzSBBAN)=YBxt0(dCd* zB^J>xL^0bqKbO!nh+Yp|PHlpzw3(sp=ANM!=cbW45c9lNQmSO)yt!jnTxOY}6^j+c zpaA{VvXN)!3(PE-`#{NVRj_Fg__Xd3%*SI(e1oHZzb7Zjs)&oaSEV07T>KU%0Pd z&{^<0&kI_+Z81I3!p~o~pw(3J6=9lY!%?7J0C~1jB$ygpBjvH)~%dUk?Vxw5bBC zjmvR<5}8KzBBLlS+msdgk6KY)>Y!nh`L`>RDalC4L@tM#N-{rf{zOcGwj1AWv)1dR?+7+$Vlg-Whuy25)@Di_$w_aaiv>B_SYfziYqoARn zr^e4}HI%LhMPb*Cw;FAR?%Z|9R+KF1df$G=gn~Xd)T`A{y3?@p)fARy$LzHh>Z{z` zdc!88%~&-@yQei9OFNfiVu(bx^eYj98{Y9DaxwH@5%3RxnLVeXvx`Z3%;<7RRps1J z_Oy8@3RKaw@AXETp*in+%Vz2~X;UXI6CCY6uUUT;(72QA#My2whtr+5o8Cc5d3Nm6 zu7;+~UOsVqcI<3i;Be=8O~=z#(1Y0}Xdx7Cy5r;6*4gaH47dxdeEpMcS+fwlzzZGP z#sO(6=;_cX3$x}ARJs_a+sm8Y`)L%b`E%};!){afxBs@6k~Dh+8qH`ZLjS0?HM_M< zS*nF$I5mAvlG|BA@6*$u#A5q#@2*HBkw{s5%Qc}j;OlA7lwo$dI&)VqYp`Tj3m%dQ3^+Wf4afw4t8&S;+)qD$uJnfAka44V-O1C_k(BmaYvGX2}s zt(9P{m!JRr^;zj=T;P%C#&j817eNoSAI9|cTUWs8_Hxs^K2Fn(uf|M=FG9esH{QTE zK7W1IEdVb@RVBZCR-b-1x(Iq=Meg^u-*by*yfuH60>Q8z(Gp0{h z0o^uVhq{lmFdN2QSgO}8bm4qxkczgdOMWJVpyB$xY`ty|XI?nYi;q0P;`td}?QoS8 z(YF6hY}wOx#h&IvWqUa}agZJ5=1;7+n6b$SPao4Sc3w9@OEpE;;a{=)4e!$H7Mc@_ z%;Vl-UA8ET5Q6%>ucChMtC$`gr}N3*F?6DxL`<8EWt-DM?XK-K-FySZl`Hptv^wnk zJhYwq_z6s2d2GG&Z&9HoC=X@%%;|6yk+T^8%Pec^VuEQ(NJ`9CeE#& zJxDB}-&;jjK?5{j`zG3MdWW93(1g%<2j$mPUHyZrKuSquW|rvuJjqy`WNfL3iq&Dq z>agSVcqk|;yrL_y7OI90^6nP%SFie~2YNX0gua)wdnY2J{=^PwRE zZQ2F=3n6e77vU<-%$sSjY%lHS4$@j?UVr-hT%3oW9@YImx|ifk#-nr|`cJ)Yvu5Iv zdEA>7lq{}JYIWH8+5C1E;^y7^|K1}5L}U8*dCADP*nRXV7U#5$d({FdRTk7yoWt-jHn4||ka^lq{O-ggPaVr# z?Zb>Zjz^w&fRqWE48>!MIIBTeve$234y(h?Pv=@p;ZH~7DZYH5C!_bw=r=ky9isDj zgBOTWG9MqpRhqlSW?X?XzS(8`+vod_3@{SPY$bqk_iqy@&lxr`BuHtp<&8)=DwIO&|k8f?#axD zzb29@U;1um#ukDXV~7(O9y`m6&-{Bv@0o}$;-~>dg9ho@g3amR!Rcl)-4*%&KGa9& zfH7@K4EuIF5C4kcOeNGx$$V-EcMki_%i$`osUj`78NI}A-$GW7i$k1q1;FaA|me|lzegtDEv zb9k~Agb?(W?51x@KYu!zl6?M~Cz+Wyc3akPhjMcGG>0GmkuGB=qKnw8<%aT`m%~=z z@m%~+2$u~N*?^*c6 z#|L@%sm#P`Tn$qQ8sO-Y4|C?!AnkOOJ zG=fG-o`3WJqo+I8J=AP?j6mB4UIeay5Q5I)J@iejf2_luVZQXQ9jH2{RHch~1kF+& z;=!MNo6yMVHII>qMX*;UQF1U&Lo4hB1^g)7Of0!}A$FXZ<&*z>f`kEwCo5sPZ#|ie z^UQ; zkKx~G{wD2aLI}aBqCNDla)0`8XPCdZ`#dEM>Yp{qQ$Aed5^_KbMS0ag?X9 z*s`~1xlz~)3b;SqMEok=!=ulQ^ND{t&SItmL(;_5f`+BMzvmx2K-UWU)YA*6aRu`G z&1=P7QN=e;Enng4z~h5_;of%5y1nX&#pe(FEmY7QYHWA{vPiA#ei{!rAN}@^Ot=JFn@(_mU3=MQ`gC^Xub)e@PT z=1j+7yu|@Lb#|PE=F4)u9%FqAcXv$*tTu^9#yE>*<5fLYIe4=`?meKOR}@&ncggf&-21hA7S|9 z`Xd|7gcNeO04%|#mXL%JqYcs$i_ONi*IrM{>tBn*oxSbPv?MyeK>M$rruVmpNF~jE z;E50Xa_xh3ceB;#?qo;pbH!NY95Za2-Ze=VL3;)*PU&(rbiXXrn4 zBo}AoOAtan1}}A$Y;A2Vji=-&;4svx3?LMRnw>2KUwtiQjrE$fg)j?JO3wA3rssEu z7&_UR6N|l75KTDkl>^6*Prv+aHa0as4B|E;t;#g6;zDZMc2K`>Hzk|0JC1sGBE-P4 z6P)?|QRXMjS6p6Cgy1JVogH_=MSXvR!IoR3!Y_?8%UFDytJ$)97d1OtaF=Gs`gndS z%vQ$gIg(Ytf#!89(#c~ zVz*+E2%)|+jQ8|2(Q}&V@d=|%G6TnYyE=9Q5-t;T!?u=p3BivIG0UVVib6?s6@jf= z@Yimlw0aXBf2r>ABWg-zerk&F&=}#tA5#o^usbmsqWNFUa;yZm^o&R$A53lG5+761`V-aB~mUeOu->(=AKQ$S8&KD>v0j|^BvOIaxMTl?qb#{DVrH@=Sy(2ssdcq$FGy|=g z8I~tcnn)>r+}G9qL0&pQS&r454yZOoyk7{Go#yl8(Gw}fj~!P2jLWv@u0%R}&7z^H zDD^(`jHZ*t%CL@&TZJE!XGCa}@?vl=t*^om#m{=M`AfEITz78m%lZ`=P1|KY}*$ z$K-jmL6yjR<2LI%2Rcsmu4RCA3$dZ8>3V^35Ba0=JnKLpjz~p*thb}%`Sp#Xkv$Df zt#1>^F93Tr8Z*y^K%NkSf9&b(JOHe_{+QOTg4nvP_05WqA7rU`jX5IY^O%jO5d1=w z;=30zO|(Kc-*-z}Td_TwxI@A%Aa4Yguc)18BM=7pln^``cQ_szIDULukDFv_F{d;% zH@7KL*)5SlkWC=!BzP1klu$$#6kX4=BC3EH6=ns@1E+y*f#^|%dZe$j^Ej*AOI{EE YA2Z#;DHJ%#&;S4c07*qoM6N<$f`ko*y#N3J literal 0 HcmV?d00001 diff --git a/webui/public/favicon-16x16.png b/webui/public/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..214008c14b95a25e4647387cf24e4de9f56510ef GIT binary patch literal 663 zcmV;I0%-k-P)nY{t8q5P!e|g9&q^3&xnZXhH-VBajdXg|r~ioLsU$1_vKkn`UO5w0;V}4*;{i?P7b6fY%nY z-q7^tvr7geBV$(uXCDD{*-dAB+hr*lIwtZoR6Dcy8vs}_xsn(Fjum%|Ox-!I=d}zQ zB?HsUj8_n=T=pax&*-E^`R)YjKHRq=06>qTQO@6gi7A!qC=#-#r-{wv#)a0Z1pul! z#pgOrBV_Vm$;BnsJ(K;nNWS~psTlpt9vvhOQX4y&bgF)3EmozKx?xYP<`iET4hXvG z-C@_R)WP6kMd~h7jDi5*a@U0#ZudR#wf76B7y!=lx+T?i{ep$+@5A7&02tR4VooAJ zbQvpjDzF(-I%Nku-ar!?iqV-&$Yb)nlE~uAkt-2|oa7fcn9-c8? zHN?33K;Gj!%<)GGRrM*`8r)arJY8Uz!txw&5M?xKz#}5i6af%$7^DHb!r}Rp934F` xEm9!~Qc8xNjy6!f5dffh5%X01u)Td5cM!C ztv(};W$rp1_+J64s-Am0@aoEg|d*_0aR&$CJ+&*T9W9u;g<{7%nG1-WBpSQ zI1a#}RVAX$mI~q{Lt6@Jbph0F+WaU8djS}{S}EOcBrg_Yp& z$Z#i(kHKvSVs&yD6&c3GW&x892qr~%Qa(A^4gfKH7Tl<#IQRBhdxil{m_-(T;ZAdM%WGy}at9RSSb65Ebrk^M9~$^;O~a2bHZ(IfCg zy%xJ8$6|Nn06@s^?H#%N=cS>(F6-pBYt;t;9PGKB_v*5F7w6<=^oy){b{M5x0L9Z? z^DtA-9RNV};ULQf)57JYf9M6Mt#EbBF-jU;k=tSbB4j-V zz^`UsQBhTx;&7eTD(=p-wlvhb3pH%Mxg`G>1890_r60gy)hR^u_~zGFx{Hg>646MU z@bJ>|{BzCQ%kqiTTRJliQNgYNFr{a|`Natj0jfVEnj8F`tidJW@$uVNem&oL^PgTX z2uQu(Wy@On@FLk@FgYaeg=Z|SZngS~AWzEb58I~!0Ki+HpV-fY>Ugt+De@Ciy`^d^ zi)!th&bC}bzb9UdfUj%?A3Yt*J2e=VB3nVYF=Ljb`~tgI>s(_x8qCQVh}bU02@Qy_ z)G&2m8o+z?<$WT;8vumvAajpM^Jasf`{ltP+m`{4A;9v@DZ}GLT$5Dt!L*(;qZ2m0 zu;wI#>j8uq7!~Pb*&HLkdR6m!dquy(VjR2kmkfdSv0r*~kcWhs~p;cLXy o&BSeUt)b&v3#w=R`@U!T7yG=7Q^KI^$^ZZW07*qoM6N<$f?;n_MgRZ+ literal 0 HcmV?d00001 diff --git a/webui/public/favicon-48x48.png b/webui/public/favicon-48x48.png new file mode 100644 index 0000000000000000000000000000000000000000..b5b6fad12519c635a61c4ac85a5f31b1511df64c GIT binary patch literal 1971 zcmV;k2Tb^hP)HU_M(tx3R}7~f-wJk0VXznaPNgo>hG_6&-uRJ_xpU#x%a#082EyN?ica?{rhu+K{EWRU<@iD z%WpY#arS=1S(5}FfBZnHD58alYzHU45x|20Gyrfqe>7$m1~AGw20*+|r4lcgk6#{5 zLn#@G6KHGe%+_l{2f=U%99z=a#U+pmA90SigF@)6v-`@t5*@M=XliOw=j9b0B90#d zaIR#(LIN>Nf%uOtcI_|7)g#E!BP)USJsl0iaGC?27nM(J_2!gZKkWP zK2=8&ds9=Bnix)}>QjQ?+9aSNk-wE{JtVO^Y#Y9}mXFFc8UX+(ofAFJEN@JhplxlP z*?Mhg48S_2Q6iD>=hHKHhUa`9hG@iKWKr0&;M;Cj^Y-<+3Y&)Vm2v}^3h8~t4;;JZ zk2R>{d-a;oL8am&k?`G#TjL}CfPX8%ABeRF089meA#W6CCPG4Gj_T5@8x3U+9kVG2 zlc$+Cb^tgTYe*PhV)TR(@m{ZIU~uGP?hp7L0k|rIsd~^8t^LL&5Bp$R7*Z082nQ4H zV;8P-eWgNZ{_^zrz%}>H>sC<+RY@V7^STFb+#G!fi{-Nngx}a~7^}5rJuJ!Er2AA= zy>1MJmLid{otC;2<3QXLibjPNmZh~g%4-@Nv?CZ>0Sa^st6V3 zUZu+XC>eE|05m1}0gV8DnUDY{q?p~$I?oENVPvZd0em%+4{=)3y0C~jCjccY1!F3eDz~V-xVUlm2GMLET4BLT zh5ENb)(|Um6d+q&jAc*8Guy{W1e5q3TOHeKxZu^_w^i2w2j3F9BkG za)a0JI$TCXGjjcwf-_9N{&cmZnkxasdzUztE{OeEE|1=nYir#1dxsFjG>q6{$DFIhU{Rn`MVI})i7UQ`18*xJM1FBa$v0nqDPk>j_8{}Z7HmZYJ$qfxg|}>6U#cV)5zZ#u$KSY4 z|0`S6_(maN(>5N=9J~TUum%7EUhkce-pfOu4PQsV=*B<%O4ekZ zbPswyr}p@<_?Dzng5Bh2z#-{#kC@m`Hh;?f8Y$=vmehPAmg zR+Z;d6BBdf=}k%Ic*nnOItrHdgocAZm$UuNlxGHMtI<~uH6J6wTgmYtuw4Hn)X^eE zdQ%__b>+MB+pYjeQD50wO08)z7;)AQMx6g#y#2K)$G$t& zypM+j1>)|*hJUc>T|BPw*SvTcPhN{{(B{&3N4(odJwMyP>SSjnh<~ya2f#63$<{; z_2{>5B$3I)efzVA>+kPb5D*u(YxKU$MT>_#d^pbclb96uCUye?BPwUH6Oo8S z$i~Jj?vWE}pQg>>q$kC6YH;&M9OD$6*v_e4V<&a!aU9!86ME{f*nk1E6{AQ7V<8Y0 zXc;Xtn&r)Vzy5)6&@7tmy_q?ueE!pW_x^rA-S5oZ?;YSax4F%2QU;|ufLLg1YBCNE zs`);jY8FDVkplCJrRPDaZ~<@_OtQ=D_V?0DC*)Me8ZBulKut}Jp|Eh=TvBNP0n?dy z3P6(qj731qFBYE<%+3Hh0JIYDi3&a^2^`NY$T&&%{r=?q2ho!@0yH!{JP8mhiRrHZ z+yfvdntUMy2*&_`gJAj-N&HUU&cmO^8LFnFRDd;W*4Sj(*g(J|U@VCzo)`oOfSU=l zTT<{!&I^aH$7v*CQX)WO{l+Sml+6Gd0GJZ76~6#q;03UUiGQ5G>&V$ejm1h*1!!Eg zevvBp9|`!HR=zb5;3f>&_#`v@#lrqzJHT)uZDjAwNr?Ra)EDP=K2-fHze( z{C&aC4qn#Ucp$n8(A3mqbT~Ylh-o_jiyl675a1?5ApAIwtUscifAH$%MCfI>v3_G! zU$6HBF+B&J3SVck!T6^>wf|`E^S1q%A{uBfcpBiMOhi%gGgfGCmq7W z79Z^&d}3LlRwgwO;DHC$R~m(QAHX~yi{i^a||E9&0v@n*i0F8}n?^6AI7=+1M z1QaVuYQ~ZiRH6gjk1w8@sG)=eXlPh_2P@(|0Huir5-(m(*=$Og#PpuyJBun4F%q`` z_4R9uh{b;+g@q3Q9#yxVbTpJf5_=PA6XJ@#uC6KDD2e|9ph-QL2wqO9001!AuPVR4 z{CA79;>jFW^ut}vv}kZWsjGr+Wr4*L znkgm$8tXS!G3dEio#;u5+kg=;_+Mk4wun0VhK2?cOUi2iGGmTRM^e}e3Plz%?dGO= z#%MD{EdX)$_mPHkJ894e2Qs_gnEAbE6Gs*Os#TAU2e2jXkaWbY--K@5FnDXvrh6)) zNgGx4l8~PTkeyIeI^k8*stUS^g-CuTnuHMrXk4{^ksd2qT9HP*pb-$Lq3^p(BX-pd zkrE8A;ie0U+wE$1xre&EK98Iu{JDn=GOijt1Lawgahg>snrx9OBtklq!C_~*n*H3plx=cRVI~5UD*9dSHWm3zx%1v@(d1h6XM5(MgnH9{+ zf8p@Q;S!Ac4H(`!l9Ixg{c_Kxj<$1ygMD)WEQvNB1D0zZp8vB8Ui|!m*SVy?aPY}# z*)#G@B45XF!$D-A%>dSgO%U$IgAYD9Mv+Y&NO0dytS~cQ?Yw;Gde6040CKgkNzDCg zDznZ$F)_1RP|`n8USZ-g>y9J6pp;6%j025h% zdC9YM<>7&oy*_qmWj9JP6dyH)dpSxkh#K^WCLA7*r@i@;GlQ}s&(X`rb&oo)?$Chi zGpBMzFCU?h1g#18F67s7%PR-&gd(5ryY%N%y%(D^{obM3fPOs-x7Xd?dj4D%gEd=g z9Ss4x{>Ea9uQFRI*V7LbT3W@-vQCudOjRXboOk!G&jQngY=i~#*C83Q^C!%#&Yv(7 zvd4L?`}~=ffsTt~S@CPS1|Z9FZ|kScB3XqG04liXp`(NCzpu9T=9x&-#+l5fPRuB6 zrOJXbGSCzNQviSztOW4W!1N&nV8Xq(el?7lmg3Tx)y1VVtKo56zu47!`l_qv>J%hc zXw1y57e8)eRh^rNop_K{jC<(#;E8va%rj=w+mSG-%lAts6M5MSKon{BUw_V(&H`HfpUSNdgTDcHNOPg85qls+Lb8@km~Aba07dro z_g?(?c;BTKt0H@6B8stt?3a6nyP7HG^~>JrYsY6sT@_Kj`Mj82dX-D8Q;D!323OYp zvf?=aKDzZ`kN_%kEJgtUlF?)>oHBP=;gq@XxcV>mww}5;c&&XBtIF7Lc`kLdox_cj zFe!x@dD~iC2YxckvOHu0B2b~Fn|f)IBb7GrrBCi zHn*k%%y9Q!^Eld?dz@WY@>NCoV$oVrl%e4fZd&j`mw&ED#Zaagp7FJ0K$Kd2T$$HN zV=Vb3XbgZc2|FUXEAUZ}0L+-8$xx99Sp_ATSp_9ySykcc>OkL>Ph{7%E0V+6dlA44 zE!=1U137OEiE|I#UpbA6bNw($MgS%P#%bwC1%ftm1r5Y#xZ9*9Bm^wQ6)2xqYn)iS zTJ$;`s@6Ud<+m4y6soY%aN+9%2bb*#0h(U?$OxS`T1lo1#pQI|p{I{{XzTV=Dk|&x z!5~5z@3W7nbw&sQqdJ0`Oqh{-{))3*XE&15)H$NJHKMRhBFq;-aYPx95I`{s-34$ux^z2ll_-}x-MYtSfD~Q;$*-FA4!}3$ zO1lR5CJv{M_4c7kJYvM0etSCppE0ruG62+dSrB(Me2?=s|0lL+^UI4l_ zlD4AqaUjxN=vX1bz{GO1rc)0gP%OTZ5sUL91bFGO`7#Js^^AZ>NamcpRy}>hL+O~T z7DG#m%iY%E1$bD5z|M1duNADcrc=PFJ!lIr(=jtWEkAVS#M;+W6+Y z9Fr{xuKKL)pPvqV7$kt*zQWIhgGmSoRW;wNHs|D>NQSRa6z6B2*i=6;>W#k%av*R= z1Q2E}h*bPZ|9Ai*L0?|?4--g|biW!b5Tiuxzp^iwNJ-kP%S7*mdLB{$RVeT18_r+I zH0Rpx+wemoA(D1=q!4s{&$dM(&tl3;l1o-0-VGT`sE6Cv*M17nQGH|j3)X_t@~=Po zq*oG#Hobir4dTkLpIRu7E6LV%UuXb_ZHu343-u!W$dB-vUeScYl3!f$jZOcOo0XII zQRLL|QIwx`;*ZZ(TPBoQ!tPhn10sGs+`F&>$Rxjk8<(0UjSRCTyMD_La__2Lc8G}j zw6aS?ju~HCaj0(V4=h%*NwaeYQQ}eAxF_7pQ5&GGuk7CqV6EmMg+W-n?<3c}3;?`N zSMQPkc=K$>xz@ShI2ZAW2FqtoJGJP6`k5K#Y#RVfHD2F;f9Ba5?V^qZznZ&b*ZOd8 zM-{;S^1*2;sb`Udl`}G^A5zi4qQK{hPabMHc<9W9L&pp*NBUrR z0TGY7!Z}S`p|-_4ANvRoPkq0M$gV@(?>19OOMT$8MGEa!uKYV*?SZwf6!rtR58S+ z*?To;4+t0%s=AiuzZCV7YfL?WZ5x)K1(47!;v}gb!Rk)=(WF3{<&tF=`~?vk&Cy`y z$8(lGe=e3hF?Ye9+EDWgB7~jWh`?o0=$x2F*WcWV^sc+A!l`D)xqF*m&8^<`O02o! z>H`mCo0iXI<#D6HO_Q~g5R-PnYN;r0zH+61^3qlrN1prXOaB8`5vFA5BAyJi*- zv8g9~67GO1)!M3`zm$lhgy%u*8&G?|F9Z&=YLvsi|JhFN6$4k0uB>-})J*)(UbcUnMuz(-*+Ey}Cj zsmX6qYBm%2qmB1;4Vq2M0sbWbKvIvX)}gfP@5f5BuVmQ;f6>kPqTM-OXHD+>pJ;Z_ zP?vi7$yX0-l!_eNAyWI}4q73=pFLqY@RY6kxqsK*whnAtxBO*cz*hj$YVIg?FiwG` zIz0*>&~d48>zBTH7N*?!MD#rXX|oeL28ICsXAJljZT;%Xn5GwEAt_zM?R$S);U{_q z%nix%nE(u?Hw;Fxc}s0wR8z`{kkrC>YVTX~RL~Xx-$=IaC{O{s4VJdszgm~jq+Bed zq!8?Te=`FRn*lTeNcE{n55T_$_!;|GR)xJ{ST9ogDYWlWv_ zQ362E#hrx2e>{O207w@*+R#9go|c35;w!zjmYVT}UQ7U>5UPt4zI|?cAyjs{nrguL zzgBL;rz8LX>uIaqHIT z4w;}!g+kezU&6pa-oN!R!D9}>9P$EN(x$fw2b_R~96f* z=}_~Ot@`pclh|-z%4-zQ;jyo*#`_ZaCDZlk0J_=lu z=J^Del>%TsRCmbQVPe{rK6tAD3X?*;+qKMf6(ysu6{NY8)C%W%Y7O4w$(FQ|X{|X@ zo|wJey{SoSwktfwF>bza=aEf%%NCa>)Ir*Xh(h+Uf}ozT51{&%>BD3$ZiVaO#=JDf zI?uCT-85`u0&)YGl|oB&gs+E?&%JK8pBcbCMlMJLwABsNN>yya{|lCgi)X~YvBU-j z?p&DvgC!;5Uszh;+fvbPre>_q&ntgsush1Dk}9&~A3xN`e>K#tiR-JYkg-XorfqN< zw$qblvPb4KwnxRK-J*OSr=!Qp6N^x@kEE7>XbANg&*C3asEaiK;yW$g*Yq>X|V| z`BLOe4VJTxhO>r|>g$*+`f>Yyx9*kECMQWw)py@2phEC5e6?=&poli6ZKi z!QADq+( z5yuS>e|U*=PWGDaW9De^aLO*eJYfuE>ikRq%2e<$aj|+j?eb_hZEm3bsR>k;ftY8; z|E?#Mb_S)2rgiZF4SD=I5rPDPU==qgh~nx4_EOL?KY!i3(IUzY4q3V&_z;>4&_I@3 z6*Cc=o6Tv-f|Tj?o`r!@U}8Zi0BGHkXj1Y)_~K+z-YSS4hOz96?D85CL_6V@0{&41i=X+jzly+5J7g9FF<`1x7UzkjqJA3?bHWH}>m4 z_*mucq03uH=EWy#0F}f!Ch1j0jJ6K87sY()(ssK}zL$=Hp7ohfz(Tpuq4K?gg3{rj z=OBZ0b+*uH1FRRx)zQX_nswG9V4=w83NQudIB=cw>X4bV`69(kqr{puIyM<10^7M- z2!Mgw`qwAiS1FWkU6a1rLDjQGKW(tyO0gtR&anj~kI#6rQEhaesh5G zkM&jVZztsZyvA!Qb5*Xx)Bx8%-?H|X%LA^G=NxxpJ_q>1@-2b3!AEl-PyZQFM9mEU zjfZa6G}nrZldzz+u{||AjjpFJLiEJ6MDa#ls+pmxtr<~N>C>jV{_oG9{0*xv==F;` zk{0WQ;Zi9JEsH4am}?gx0HVcACh2d6yHQNXU?R2qeR|R`41hJb^meP0ur$&|)v`?V zX(gUw_qQcXesTlKyVD2~)nLnfe%Rurf)Y=kZb==EN3X~10OVr@D&iMMYa=7G^cl~r z5s|gtiYn=$+RHL-joBpduyI8_tug&&~YPo10_ip$d}$+c{R(AzBTS4y1=i8_3@_ z5^{X&c#K$yN9EB=#J#R%#<5QzfQZh?`Smk$*`E+8yv1VNknbWIKD_H?vzmTHhVHR(QvP(^Z1SX4{e7LTUvtfD4M4VB}7Y|4ZW zw)>z1E1aL3bzR;g(f!Z&D+%+ig#FaQSnW-m&}F@S{m0X;twTMn?%0yNS@h*1%;|9> z=IH8ieXL1K?mt}@f%gBU-$iW$68}y=bMWDd|CWB#PX9_jr)R?tgY?j&?B^jJ?Tl{r z_MC(1h!&>+)|SLFe&NWpk=z^6;+@PQ>aG^$_(GSZqJry%#*QU#K34UQ2%yW$e6{~5 z36=~>`{?eyOK_w`*Gh-q^`}g{@5%y!DQ&DAWuJlsZiafU+j_=Ur&#>=FwLbW`D`zzE$^J}j1Iy-ksF-zNFO+1~&gA62I>14nJ*k@K4@8NC8W?tDE*^5Py9_&) zcctSxrpzap*8tW=PSVR&8%G zO~v|>6R?pqtU-l?nk#~z`n!KDE>$$sWDYVhNqPe___S;iKJ|w$7x}s=#YKVd>uopA z#_H7gE*TCGAe6K5W<3TA_L-}K$hpFzFe_#2I$MV;ZoM2YaIN@_sw z(lg#CtQY|m6^B>L1I3*54h-A`VMx-aj9b2dEW4`~%-U`zl6lkW3&zHC&VG&YfCp3~ z5>Xh~$5L|I&d+#k^AKjt90Q?dS+l=diVewEaXQ=DIUvgH_pIDEf|zOZW^UI~KN;r= zfyB%c=prY7xxtDn9Tmhn|9m($HEZyo!>1x&f6Yp*WtmkTXML%GXG_hmlXV=Iqm?C%pY0c83?Q=s+qy1@I3P450TBG~_C|NIbxmioHAIpl zt~W$pema{AIXV7lwk7-#_w$B+ZYhE9pG&&sf6|uNJ}$lt3YJM?eEM*pwk%3c5a1WR zWG|8;Qe9AAQ}cA6s^1O%(6(6ze^jk90u{xwBG8Xw9qzSD`zLJUcS*oh+gj5c@WWzP zyaSW;{heE(^Tza)p4!J4j|HzZ8Vfz;68rYbnh;$vEO+ z*L~woEy=4C0I$%|oV-iiLhe8=$+p@_D%2-m1xI6LBl*nyHhdRIP@fSiv?MSU6jzaa zXdYH))ua(;pszY@h5im%fRsnrv^XF^oR>u&AH_01qo8rcG5)aaW5;f}_tKxh!_2 z_=#j!*YW*1vc<3cp0=RM`9T({_I6o1a{ENVZQd!qKz*oo`@pULZ7O4-m(rAwiEZ>Lu|LOxv5`L z2zvArU+rdTUBt)6)GILFFshZSNUAyX)|$lG zVGwR2k;OYTZ90z|ExPj&-!+)GWp(W98&)Z{%Qe{SS@Eks*lVdFrNSAzqxaJm_-ibu z&AG(;wn;lRL0x*WUFk$;c|xu$**n{2>=6EY59vKZR_)~cMmX(B#!xvI z_nR}wz)>ye&P_y_ zQ{pTMdpg3Xk$G-ee_AQr(Rh2fKK{jA!}|O6vt8{}^09Cf=`1ku(#+cT=VA4hhWmdy zQ{o(gU!N{Uf620&5}$$@m3pXKQg7ZYoB?IzBx}OQhA~%UhA9I|*HYNk9br83Xj3LDpfsqZ1bjRw8|0#+Fu9o&UV+CsMz47R3LKRy>ms z2>k*!d^zX-28v_y?T+z3yxcIVQYVSzmtug + + + + + + + + + diff --git a/webui/public/maskable-icon.svg b/webui/public/maskable-icon.svg new file mode 100644 index 0000000..fc47931 --- /dev/null +++ b/webui/public/maskable-icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/webui/public/safari-pinned-tab.svg b/webui/public/safari-pinned-tab.svg new file mode 100644 index 0000000..ca24178 --- /dev/null +++ b/webui/public/safari-pinned-tab.svg @@ -0,0 +1,3 @@ + + + diff --git a/webui/public/site.webmanifest b/webui/public/site.webmanifest new file mode 100644 index 0000000..9f84c23 --- /dev/null +++ b/webui/public/site.webmanifest @@ -0,0 +1,30 @@ +{ + "name": "GovOPlaN", + "short_name": "GovOPlaN", + "description": "GovOPlaN administrative operations platform", + "start_url": "/", + "scope": "/", + "display": "standalone", + "background_color": "#25282a", + "theme_color": "#25282a", + "icons": [ + { + "src": "/web-app-manifest-192x192.png", + "sizes": "192x192", + "type": "image/png", + "purpose": "any" + }, + { + "src": "/web-app-manifest-512x512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "any" + }, + { + "src": "/web-app-manifest-512x512-maskable.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ] +} diff --git a/webui/public/web-app-manifest-192x192.png b/webui/public/web-app-manifest-192x192.png new file mode 100644 index 0000000000000000000000000000000000000000..520a2b40e6e3303183eeb4cc441cc12b25399f8a GIT binary patch literal 8386 zcmV;zAU)rSP)I*$6ac0$~!803kqeSd78Kc#*B$k}O%WwePy? znLmsP*p{v3U9ax<^WlSimvbJ!_x#@Xoaa2xa|HTgtEp+&nzE&u1WK(y)}*AU6iBat zTnS#lsk2QP^d%->PQrvl%qc?l$dqW8N~*(_vh8ngKeP`>>Sc`3x`V2!rVYlqoZR`)84dbw4|iOo#pp`KuGaX;GKY?ML!wR9gzf{ zky8F75uJayx3_mu^&Zr0tf8U77fYl*ju2l2iqvd7!+J-I2;|*~*y6W(dU~c*YgLU# znwy&))3fuRL5RNrf@-vxA;S`k3;3Gf>%Oz4r6s=B#?~x=noXN7mnrTB8rRxZh78L` zvS66g34XkS5*45Qj zBvBp&cC6<688WEMt0|lKV0UY4*Gj**Vgc0D*Iy&0_&FJOW(HFsrWDB^x3{-GvyyME z$URw8Uw@;N;#Xt_e+JVbvr|GoUQ=KH!IgZ=w%iZa)HQrU!cPFlifzk~K_6@&{x}@Y z8yXuOd3BjzxG**K_3xL6`geW0M`dYk+$)%k=EsW_`KBCHoP+>R##-OD=+|s;XQzDIO#< z_%m1*nXTC6|E;R3T5|TXqyQ9~>keS6CU#}Wpc4|=EbPwzz0~{XoC3Wb-mq(__cLU$ z7*dkDs;jMS*O_me^#r)`$}8gutEU!SD6s^#<^HeAsOI?+XU(g2sTI#sIvx@YHLI zDL62hVEnhky2tiIDmq@q(uiI;g4O&crmcyax1pn3_lUa{UZ`Aq2VEAqpY|xLj_d%`t?=fSrG6v~zqR$%6wi9vO%c zORXCrC@78_fx8rdQ$?%NA4O4!g!0)`(?og6hIKpm-%OF)#^Z zQB%|KHmS&$jkZTCJZ>+gMdf%5n&V#xqt*ZyqGLA6H`*3BIF-JOR8LDj+M2Hbf1w13A=HHQ@JV64DQIQb-l`UD(;Z*TGPfnrORjdEG;`BjitOi5u`#VbI<3;e9 z*aUp7(My4e{G+C@LFN%0&qH|9@HTuzoDSKnr{QljPUhrzkcPYG2N4Kaht{5Q%dF#A~xSlvFB7Dv_YdD{&xs z=va)1OX9XElzIfkZX5P>tchDHT>RpAl!Oko*mV&=Fei+2o!$Lou_)8CQ%p=vu&_8! zGI{n%EB&wk|F>tnE;zMITJ5uQb-sh0K^sNx^|NDMn5A%Cp_5-7k7?eQsIJ?jsm_ED z)K_oDXw0#O_E`XYin+!XI zT5G0fCK(v&CmM?y(tlBw;I=vsZx5xX5Drcy`1sygP5aRFS?Y0n(sJ;};xRgU+UV}< zG;Ht#9AA`t_O&_wV(%;`qUm>t-R5jNb$)HvKAH+39ZQ=Gj|_3R;{bE>Ge(>82K%QH zy#Ix1e$uP`qET<&tFcQ|GXV%8$juIF)x7Q`l1Vyx+88=9fHdI&(n$$-br~jO;TlIX0fcfx zS~aaJvvV`FceS3=k8f6{6OuoBd71-ha?1O41=>x6G!cL^%S*-fcM-QaNLXm#oXO<$ zBwf92rkwIzh$g$6<682o zng~~2#9}g{JvDqnY+(~KUPNH+u96Rtlb0-IJnK&bCA-X_!ueAsN zX;KnC*)qrdJN#_$npXrT?RJjm=TcA~rY8UFkohYvj_}y-Vb%LpeH03zkY84F&i7p2 z098BR1Hkmi00XUiNKTF8Fp(G%$t0aUZKPy+EFuA7Qu2wG8UEj;0emKkvYQhMojHCS z6+s&6aw$Ad81#;YAT~viOsRWYQhnrw3d?c0E?o615Gi5Pl_el0C%O+Y(!C!MjbgJI zdzb3z>$KE@cezZ(C0{r&$9K0HTY)c$pu_7XULK@=Qxolc z0D0xLEBk^F+1o!twxurt?z{Fm= zld zs8W@--2PmuFMBTl;{)9cx9uf0KciVCx!z;lmUMTnl!S!;);iBU8cl_w2#y4NI5&pb zRF_Lc#q`FlwYfO#3h{(`>oZja5Gp9cra|IMuw)~_l8s22;`rgcjP@SJkxHT{>fFVq zrYD)3*EZ_VOtefScxgPrB`O_OkwVbna}zHMQM)O$?0Nl4$#N>xmH9c?Go@OastO=j zSfOTfXCZ__al>|s8@7{(F3^AQMP`S3aR{-d^}^tA|Ajx4X8cq80uS%>uc?gzr;E{| zT*@25H00>BuBhK&VgcmpSnkm7a#OkOY5-=&hB(poD$&W4tn}V9Jv(J+NV8G&&nMYC z5ocTO^5blT+rdoO&uEU9?2-WG!7R!(wDDZjm#bS(R#gE6g2p7i`$L8J-+ny+i&JBa zbsu8(WIu8l?#$yu$Fw%jw0yf`kq361M*t2RBIG4m;AbK1!=L9PTR^scCZ0t_p}KXY zR26_fWc)!FS=k|qw_X7<#q7yJrUtv19qvb-V`D28kFpTcCymjJbj~I@5lxYQ+9PCF z5Mdvp&=2_m2)YqMpa5pcIjblzsQ|J(ew@qL0MiYj;L9(?mtRUM5odO|kEy|K=EjEL zb#Kq)%oJ%6@32ZF@Yp2$r*{=03Id3b2TUHT74SO9^*foEQ78FFH30-d=B@jq*c}8) zstA-+5nq^P`gjjh$2uA7X*1e1!+C6Qkw0&=*161O1#Z;@pxSPTFE>xErZmOj_7U3H zMBbJwSWM$x-{sOb5o364(P%SPL`8vmdFJW~z+&l-$9oSOZGvfd;n29zCaj2}aF%Lq zsU`rAKUb}$bm7>MeMXyL8lHaTgwZCfh_H%#4yp;jX|Z#DCr8si?q|7l9JAI`Fz8n= z+CVh{I4o59P^OZ^=I4wy!!%6ICRC5J*Ah>bdft201Yoz20A?meS=av0Ge}9v;7RN3 zhj>*??^N#z;INJWW~WAsHo$`%FPG^#>!g92@(EzGnI&4}!bBG`0yvXNm3CJ3z^US)+UBj3kwIt79Rs8l zstF*Ow9XvT<+j$Ig&ySit@8wr#nnrRR80VhsCD8=thQ;V2LZ2nYBODkxkdHXE2;?~ z9pTJG7m}*ArJexd)=4Y$WvAc49c!f|Vx60UDW3od3+2`NbHk=j z{Lm9c2#WKpBmfhx0pb=CfKY5W)9~!F)spSAV^gtykJika@d*&MSaL~jM866aW~a7n zZRJt)W_((`o2_aBSeThqt0`S5soP?-38vxdW@}{gon~Z8J(s&`0+^k$R#s7Q#U`Uo zFb$VBS?f5fp;7f>xl|Lt%(%6(io7{tTpnvZ`n4bwa8eMoP8=6N@34Bg9O?;R!de0V z*tq!;qfIcJ_g!9OvvX(PoYBsKF z;GU|%aMMX!ZhSWX2MlYJLz1e6J zbm#V4>WnsFRUAI5?uM(X0vH?aCy}sDX3=SiDjUcvwcd1KwQQ@)Wki9FHQu6C2U z;PL0ubomWNo1r;)<(7BreW;Oo zymG_>0vI3eCmywK-JQ#C`YW7S>YhSk2C{t)?)cA5Mw_u}rsorM^s66nthxZCOj&js z1mJYJx%MNUN3l?umkZ%^DE#P4&3N6`$@@K%J+0%K)GyVfEXqLV0WF%kNz!Aw{;fA***sk{{42#-}z7T*zO_C z+SW_}$9r1SrUqvqQpo#m{VI8-)kd3gL7MAw`PDZrwfvm_G~?58c58TsnPvh=B@^@? zY0;{A-Ep|wG+liiTW+`x#f0ku#b#scAHAD*-+Yawy*+30=cESn=E&N03V;SiU7>?2(Co@;Pqow7}_+i4kgurqfIoB1A8E8?||Nq+d7UQPSZQ~&^pc#N{z zi?!@Rb0+7z$tgcOcHSQjQhWJM@~Sp4Ie47K8GR1k%PlVCvj6xYns2-wf9PzxoPhu> z9aCJAZ`~bvfbTriMfae#*Hl#3ZPN5vDHMh4KmHZ`!F(;d(TtJdcJj9_&-S^=DLS8j ziJq5QSeQ}Q0pRxgsJx_^>dP*{o0EOP&r2z(8Geo%YOGVX{>>aaxxh_d+O7HNm6{6x zU}MwU*?!%hYTb?M%uJlXS8K;+UwN4H?8GEPM>;vS?;zuYL&O&5k<0usg%CKs9)iV% zlwGu$!cEn9b8}W~Lv(JQk7VtpeC1y?od5oV!~FUMZRh`>lK_NbcU*wN&e^FEK&uMh=m&sI;wwIrx`5Nn_9!}16lUJ@@#R#D&`18W}^Yq=B z^%RwI_n}cf-DI80eLwn5uP%chbUY)h`@oCLTcm#I$Z$J(o6@ITN}^&5PxLsAHe*=~ zjVlh?tY}3SVFz} z{})2nnc>kxFEM_iU(YUAL?X6`5?sLUOrM?9)404I?mtn3j$>=ZwDwH%#2$TXOX(^A znM$$uw+|wvIe&$TxgPul7j`jP6;DwKcONvTBO;lSeEorTQX0EG&qR0Mz}QePJqLE{ z-R%V#8Ez-PA${ikOA+6&g(term+kiRX9qdbt8U4Y5Q_Rl(*)+piS zLKwC^(sOV(Lr0Go)%_bx%y#1|u;5nd8u1jBaL*y*|J^Sf9OXAJ>gV=66XW&=rR3$u z?6d3ILd&nPm5`?*plS?E1{) zNrH{$^oZz%5Q10T&Gb*{zaEeh{>P7xFfwWE@w$dbh#c=eMEl-nbnP=4k6{a?r@w!t zIK5teGPF_m^}u~k^s#H7HkBS$g5d%H*!S!&7#Y+ryJ}*#8()6<`ukUkr?{N&9?~yq z=}^yh4vgC=W}-+AR0GG#N85@S^xlvM2u%2`sWqb z867*(P5$Qf8|R}I3*}pRs%K?oVV>PL!sqThz+x<^kyYxUd1W{?kTS(U=YFz0eu9M; zKICm~dJJE+V^#HhGQ1Tb1pP8hMd~=&Sr`8Nqt6fVm49g^ZkAZ<3t$NW03apDyAP2{ z#3`s)`jqozB2GMO1W(YXqjBkt-RWfSafM5R<0$8eoBQ$Oef;CUwi^-KxN1y41v-<% zd!A(PlS}6nPfT@Y=KUJrDK6)Gt>^ZpB~y}rxUY@>^H7)0R;?XdC>;3`V5zyn_;4Si z13eU1HDa^>A%10asFS=c1qND{-oyfhESwmiGUpFBpQ+h6pS^QGPwzcxpk=EjY72!U zp9S1TTe4DSr$*^Jyq9p{20VdW=BCH-S39veETv^jA%vhm8G2(qaQNsnpS2vmA9^L`z$JDuzufS$dt@#TA4nVz#|_32ZH z6Shz|{4o&LtX&QuCBwaK%uS9_SXYDHnc4KxlIYw#Fa7*+p5E1Ism1O&k@VR@;k@gB zTBB`QD>EZw^tQZ4c0nHg$oeR8qBqA6xAX7!-p9niaidL92O)ObLXo_!z}teUGdDg#WJ5W2r!@+~nw`a&IbMF~QT9Fh1c|75V)@sKfM42zk#Lznyw7N3)MR3C zi0<95V0SnOmK7s}d5S<;2`MEMw_A)!IEO$ zcJuoPmzNrCilrDI803|oJ<4d`F{4e<3@K!@0BqH@n?@1V^aL#jAp{$?Urf_`u3gso zTI-p)iAnbV<|%sj?n7FBm3tl}lb!8{!*(Ev;5pz1qixa(DJ4C7UZwBVgKXTG8&y#hN-t`p z?#fFESCkoTj;@Rx?WcX$3-rHsD7~jT=R_pqHcEP1TZi9pk5tz+JPGnzBh50BU`a8R zJGN53wTUdB*JyJzVR2@TzE}6ty{CnVLE{S28jXM_I@%6j_xhhtQB_}mi;&_c#+hXf z6h$Ggs*(*mE~f0FM(hr2?<^-135MIdIr_@096QjuuBJIlBmT6rt@WqyCIM7bRAf1_ zJl#Mhjp}ur?ktKM>nLg5L_uv8PWLi0LQFt(VS(YcE(Q-BVX(E0*utXGCYc5S!}GJ# zl>-9<3$OqE4^-FI--ZzXXpCv*MhHQ6ejee7IHO1V7;fw0WN#m1 zy~i?l`XvyE&v&%7-tmXOzlo-%ChtNl(F^E%nzIE_Yzo2B5<+FghCTW0P$#)L^MV`8YRAX>c2!ZMs#kD z`N=8f$0tt>`jewXXN@~h#PUe_na=hj-+A*(X9?iSE3dQ<4xii$T%?;}88T?kp@7%D zwWX!yEeQ;#5^NZWMOe0x}aqnZbHU zCKZxf+uPfhEKPmhxIzP94x zxb5w&&m=_ZBH*Q!em6q~Ef9znnN+r|bnwF}i{eg>j!p%0vL6tNEgRTQW?hiM7$id6 z>9p~o_V(7%)iki?fqN=!BK6N8n%3G@h78L`9#Ew6$@ccv7uM3sn$HL5=xBYuxFEk- z3h_B$RVvJoVFipL_(A2VOH)&mcOe@8bA)mm$YM3y&amz|Dd3(|B6det zSJ$*!jcPJ?uc)ZVvb#Jt3BfHOu32x@CNijnNDAcdB;x+~;@l&>y}elS!Epcp0BcD^ zK~$^h@(QLIwYc9Dsq3h-nwfjg-DjVD_Sx|{`+P#E%Hu&OAs`S4Pf4_`$a; zTm#EgvokZNCYLI|?Oi=Q&0k8%vTnZ9J(~8b{M-KiKY-1;@c(E4o_F9Z_=%2tN~Kcl4nx9R_L5|D?3B%G=^20Y-Z10Ebh?&;B+ zdWq|L(VIxzo+{j)R~)kHxIGA7X>}+ADV>X9HcTn5qzcp(8zDljB&$yUE zL(8-lj&Q2vVIcaLLc92J$5$!k%%wlA#@(93-pw{w261ywRb@~$cpTXmz# z8hiD`&yU;*gjDGmf0&zH%swKq#-4?l|B{Pb+NxSeZ;fV2mPyM7KK1{zO>f!pHm4Ru8qE5FWR$e zbe3BdlL3t{^%6bijqzKC9BW|59wq-8|rGwKs4t%D-!w%-6YX^t0!f1F?p6a2re}7h4Ql=3{J2i<6#cNRg zd$b_Ru^peJrA@axALA%5J3bl3LRKXjx+4akK zJlvsdGJMy@1z zDrHrT`0TfB@>?Y-=r~#s8Ym)nUq@a|zGL#x`Ce21$Ei|@rA|4t^EI_Jou@ki#|&EoXepf*c`FtjmPn0at} zV)Tr&Xl3F0g8k5cg)WUGibeE=eLq?m@&&4<3K*?F@G2Qf2Yml&QGy0jBTd{iKr1iL!H=Gd9g)g>t+Z0UO4#Y(2*1st zU$M6s>BAHWRMAA-QCc(#Eu4&_crYj!{^*WFFAzcEhZ~RAa)&onq(_mq#B!k2&Ke_m zWt$%>gWS7f9kmUgrM3e6-XkHHFWt`5?%zdz5~3nb-Opw$#?h4+9<^`4zr+`&ah{Nb z%J=Sd!~qL8$)%pQ&lrr?iIbf&&P#J#U)PBI_VDKsJ|5YGy)QJu`t?WmxC`E?7mYtN zvw|M|RI4lIRNc9+bl|Aj7}>~k`3I#By^7O3@|=Z0KOk@R z#~+qiC<63tfLq1e&jmIXrifaKLx5W3HK*?2=UIZb{2tdkCA1KUB_EoV6;Z23A)pJU z&!BH!;f7Z!d8AojaS2apaB=du+5Qo2cYfdnxApIkv_=f9^6yO97BEnKjlQG5aR{77 z)vGj9X8odr#X0llT>UJA78zu)ikGeAT+~?PL5J@%$384hV0Hv7W)_C6EQdbTm)cU` zVl~*<@8tSN`dO2>Ez720lychDfR4tXOgIl-EWSr}x$?Vk~b60@~% zGnCan#zNqw7PkEMHCEj}EvKL?4YJ<_48#DoipV8SF*Zbk#x`}wanEzEda<^U8?Cs$ zrL8$8Ci$ELE7JJ?S0&OqjwCu_(4gFEj)lp2mzVkq*meeJI>9poA447@r3E>qb`YS! zK0wm)9rGX6P;eqQy6?{Tb4}vnwp~VF;&MRH-K|tQqq|uuIglscnwhxUSg@yLL?{~K zIWQPuUnd=5fFS~gAYb9FjmJ?iabfV+Cz#oSysy0*ycRh0&L(ltAlZ+Hr+*)>z&wdz zY|kJPZodtislCW{pa$yQP=xg2_O!oM=f|x2iEPkUCaA5a&|3=uX^;h2s2>6KPGps52MAxx{iWDWoY&y#LB1} z$y(DGGLg5}G|&3ERn+9gGxrrBav&L>^It6ZtDSpb&DrCm2bIBx z1mU)stc=e@V#htx?KTX8g%v)!{($oSX_UYUIK<$_ot&TPss4^Bm#F{;&3T?@MJKC+ zW&RYYN*EURSaHXzf&s%D{J9bI)?eqc=S~F>7oo{R)Rvd8F+RZ#^q7tC3iNdqP%Z?h zNQRdZlAVK#CGAk0frMO_5XpfePFAo!A28p;DIwyA{W*5cc?D@XCp@~(IMB868I%JZ z=kj~*^4=f!IH!N9?6c~f>ZmcJD9*(-Xd*DPkwDJ@QTAHY_=wDzP;0q{@=vfsi5jrD!%E}HmEvn`&d*+LTA9~jQ&TQ@zVaW=?))ZD{hSZ9LO zOumX(kHp#2ZH^Q{01Kkbvx7rz88XnklsXM1m>!ks4zXrwO&<8R&M;h$bY)$M^fbhPDH$@?cJU zln}uGRLTUE7$(QX+P~ALam3I}Y79pjTN_zU<3h5ngT)r_wCN-Z-N+UjHQ;q}^@8?U}>6tLvs@8wHH5 z)1qds%nO)qB{WjBl3^eYsKP7yFat8VV+lzb*D`3zGM3e(SpDSc%O%cZqsjmY&$Y;dnLmS6C!)0D@?RT=B7 zBcInHUhn?xvDq{{95pwu{3I6n3^3;V+MSUZrb_B;B$M+<=A@Br_a-TwXAP#CTpC)Z zTUkUK3OT1)ZYRjf(1ZG_JE#5su(ziz7t3%urtGfO$awNKi>V-HYASNI)4}tBaTt;c zZ{mJxJx@?fP)d9E;plZ8EGrsPlvg>UIMzw+<=W_nWHCxN4(|fLZ1hFmmF-7CvB1-L zcUWNxHi+tx2HK(}S$HTZiE^lq2&Gzo?agP7c>RH%iSz7)Hem7RnUZmv=}B~6=_sMr z*?>2t!CEof7k78^3Poe2>O__&7q*U~@yhWdr<%O7cneL97@h>TZ=Z(0lw8MobussI zL&@YhjZC>-BX$v)7VBT9muVcp>XKznGchqvH%wP$TxZHrfU7pu2cO4(JXSjkZA#x* zwWGtr!0Q_Os-n&%@0PEb0JspMp{gGAaB{4pv)i~gLb0DK%PXyFAk7;qGaobjy~9^& zWAIt#W)0=O&^;=?iaL0w-G!5_B-XfM-8WBgK@ufvP0GwL?pl)nUHP%RoOEMbM|rf2 z^aZ?m;TN}$nC;RrZU(ei`3WxCxA$npSg$8(oJ0k0npCayqgiq%&*Jy0Kf=EGM9Jqe zqF_e8&gC0Z<4dHZObC_(=B+HU2y!$nFeU;INM77lw{^&N9DS_=UQrtp_c1pWDu3Oe zD@!mjA@eQ!OyGyJ+c`Kcpt0si+8FGOhlz-qF~iQh)AL};;qBmU`7WYMq?BJfwQL~i z>t6CbIbM!BoiOCRL2b>sw{F_;8%-Bjkn#Fke~0X-{xav+%o*Ji z@Lsgm^*rH&U{+80ALC9OGw(%CqsLi^YCZC_R@x|81?>Nhein3dWxeKF=MFLOVeE~M z&VB`q)=YoIx@)VlO{tJbfVpz(RDVM9>zbU>yQ7_{5?Eo#<*q7$O^;K;QYUnb^Sd#v zuovp^wUxhpXAol!OTiUnMlrD*_WuCB(MR2QFguq)w zpCgQ)e!J2lzJDOB^({2pSt!=Eo-vTz_9cWyg8j$VZGbW)?3`*|I@i z8-JEu8U+_CA52rt%a*@XdiPH9XvF0spGzq|ub3L-DCd;PbpR~Zf}RqG;;9=MbG<2x zT&*rm!<BBpa!zPi}E}Rj-%$Dx6vWq>exp_5lwv#Hbjx}aQ*yx-z@qRrC(e?VlXc9Xv<5* zJH}E6$5(jyD}U+RkvrbGaLoR`Is_gq#Q)dlQG_(p30+fSflBZZE2+ajJTTeZGg>0N z1NEiWpWg77ji#GH9AAAvogRnE$a1I23S}JY?nd)*P*$V;PI?HhtZA+O`ki60w`aql z$6af`@sf-3n?5qG98kl1!p_TJV`=UC7b9l7B1i$xzlR!}Exfl%?sln@oyih$1tm>S zzku2AQ7o*P4d-*Nfr&i6=mGubB_)i#Ro2-b%+|8#~vI zd{aNW-BQ%z-ys04b6w1=NaT<^Oqnb_;fxkUbo$Kz+ zq*tr5q6=Cqfqg{k>BmO8D|&3pTjNW!GBk+%k|(9IyRHWd+whw3y@!ZH!9SV${W-OU zl3>T4m6wRKO1mmSUl+fpvg$Wlj;p^*_J8L_>pVQ4uqgRn#VZY$J-b_(t?l72)vY{i z>Wx#0N*UHDDh=%P?9{7N%dreO4D_*XS2b-F0w%k=D!sD)6m2Y(TQ8%Hg*G3?7XOjM zL@dSBbh7%hl1=y)Y2nL@70}PP`)Y^z4#BcPk0+R^ncQiSPSMNYtB)mzbYk+i2OQFo zL^BMI5X>qP-Rz8MK8uI?`NN%#Gs1}6nomC_>UC>YWohyXne}2q`L2yjHB6!*YL!qr zNU^m|kb?TUoF(($zY9n6E)GVHwXu6HjYe6DeEY)R@F8aGnTdy3f%5T79v)sW3VoAy zujXf_>-q(#XeB8-J52P0u-n?|#d6LJ<@s$|WvHm#tNf7YL&?zq;gn|h)V2iJ5nLM7 z@XBWmoF*)>VYv5NM{OV|a7BH2M~%g@&nQ)U7i;~dVBV2a8XnpJWn?pnJ5t|RGE2$r zrwC|XvTQG{eL!$e+4E6;eBt$Dqi=iUk4W5W)`#6V4)5YfB*x$t-h8VI03cTLcep<| zz9gJswNv>rQfTQ$eI?g9$=>H(zngG|qnMIgNG?WH=Y4R6?;DlB-nxs(%0ObsYL3Z& zn$V*$$(xsh_o>bMJ9(K>$*NZulx*&8e?WfT#zEa$>6;j3GNeh?GN#=2>Rp}~mh~3B zx5mBo>6~vKT{?Ps9*?itzc&`%+F#%rjeHg=1$G=X9=#CvTgCMhmR^F&mUDfsjLsx` z^+eYN?OOExx#^o~GLFQ;{0rJFhhpYMuJwf=wkVmeDafP>pBA-O#zK(|VcskU0|6 zRBOKCNf6fJxY9K~mIjD_kObiiZtY*1Ida|ukT$MNcO8#^Zn&#JJ6P`1Ikk` zJas5Z3XL&U2kcF}k^dq2P)#dQ*Jj6ak(b6k@mZCV6mF%s0^8M1zDEAWmpW_SRNp36 z?9Isyy1lBD5TFG843lWhVY##@5gPQ&Xs}XZk4N6>Nu&n<@bvj&jtrX#3~GOknPXq8 ze-f%t(x5EW!y$0gAmnS@mT~z7E`DWdIaXV4vcAZMkNFLRiAd>Lkb+CS7^`cv99wt^ z8=4rL3;i*r-TbxkXEI3bPU9=rFp0ee=w)Jt+X9V$MJB$V#Aeo=cxmJGXS8A*OoV)@*IRS~=bzu#Em`a%&o2ZU19~fdU(|m)-EYN`0Bdb&tt-6ic+|-nZ$02)(1-6elnIq5dbORE-EaJ`M(;Jsj>@2c4ItaZoL18R%!d8S_&ZVxika!Aoul9U zC9QO`f6_8DlmlX`g7f)Hu?<&|2DbFAa4K6cw|>=2SEw8ric?V6s@D4TXCJ$u8A_&e zs?lkgkIv6(Rv_i2GjI$oSlj$nKe;6j)(S!^?sUBR5}l31+hcI=L>idJ0V%0(nwn7I zZB>Co9hC};H#tp*9%!JlcFv%d4APif0*>dHmGKY`FM%DLL8;pG7jsMQ=Z_@Rk9m%P z{*b42Y){|f<}>`8Of()ycAOMvTl*_zIHu#~pw+7s5``i}?iYF17#IQyNEa3qp;K5n zq=(9L%w_C&hdH`k0%8b?K!8ZhAjP=_JA3&IotGw3V6BNY*?-K!2B?P1@b{S6^wOvp zXuu8LmvZSy45n8Uf~ABGCWASk!IU!m3Bs?iqzId8upS<1Pm?P(cD>FY&s|_Xsi6`= zAx~mris@Uq72#<6Q+=N>c5sw7s`EhqUfqcjCd}6b}p{wnu4e^`}x8 z&~YA!O|k4nTr^0SPb}*>jpQe4Yk=H8ii9i!I?rOVL0Xm;U>jO3v&KKnuKymiJ;6k9 zX1(6z`ChCQ<=Xm84M2Gj)ZrwhcmH}bhuG$6A)}BOa~Vzb)H^s4mIKf+ELf0xE0A7r z#&<41PtcFv_#du2<$&ANG4&Rk0K8aB-j9=={u8k&U6n=-$xc@dM}sA)nmtV(ER%VO z2D9l_{R04Nn|k<_aqfVZ_(z}I8O=Q}7eAD<$nwNZ z7hgQqodaSPB8BfDp^Y^pAuzFX{D1F~gB`Wl{FEZ?C zK{aSp0YMW^(&#jppae(~xZ;9dt8JbGkZSen{vDpub4WHd&^-)LI%?YZ61b|!qiYD7 zm%Iz0ak^})i)rrGX@JX|%gA1kfCsNxd3gzssXYP~s20mcA5ON>^N&qNeAW0rl^@ ztoV}4^EEqF-_rQDV3r0AGEMj>r~<0gB$63u_^D@73Q$@t7xu&houC0L=MQMGa6$=L zFFJ?>-|cU1qf+{4KM++mXmAz$m*lOKF|c-2Y5q0<$(76r^nXTZB$h!BXl`ry(!Xa> z=eZy@9G0r?mVYdhVj@0`uXS1=v~ABQM7By6YhxqSLfxhAf?3TfIU18sGyHTEB+vmy40l#AW`htz6boonHY#J4Mur*FcDzr zfFzQ~=^QkIZM;L211mO8&fdw-mQ-)LyJUrTi`jUb*T76W^r&|pkD)pjbGXjlshYfH!qy$`X z02}bOYNtOqJ5a>o@W2Bz`V2~ozx^Y6I~xF`keW(^9lu>21G6I0lZZXm$d^aW6X z9KfS90gEP`Wopx@Yu**&{(bc2)EgjkZ0g^7iB^mYcvy~B8F+B+f$*c;1mT~e2T}WM`~vn*9MyHN1Mmh_G2|s|A1UI{ft}WbHXW*bxoz@f?cZ@v1OW z&BI&*0hBHq7IE9zV~iWJ^frw$O-(sY1sF~ah@ZqTg@HzL93B?Eu&C#)pyv{~`QDN| z!fEqXB@T(P2?S{N{&rzN^p{7m)tTV}h-EL}V%*v2(dA&57hMIbQ>Jg|jZ!c@Pg=E%k@5>o%I`nRb=*6()HK?3d6>3*7w`FlC% zj&?oDdV;mC&u%mDY}*&`Fz+LY-6ja3c$ifQI1n>zT3urce07S^>1;} zyf&^4mwTwV3AMcO*->{W^B^R|Q&1rBSv97kGAzicY$O`c1bpVw_3Mx$=8W8%hnOn@ zpMBp6)~VGgK9NN~DxeXEnlTfNB;W5#nq7*f5M+e)WyPK0k?7}8l(6UNE84v}JZL9B zQS7E8n;bmUp4xxCCH;{f$lfLqNe7Jq)iFrOv>s&JTGthF4xpb4hMB$eW<`25fwRH^ zm;&hbBX*mp4FUXP>g6ioi};~r)I6<;+Ja$fyBs_?HTmFBc=?gemXfPH)turM_0CoB z&#N!^xy&q;KvEU%y>qE!WT#{W&QYHjV?is*LAE?fc-NDa4FEE3~F z$Y*q+gpUT}Q+3!xTpAwCHjeAJok!OxXr{aTt{Wxa!T+oJ`ndq;o!jaS-yE_pdo-e8wL<76sPmQQFq(A^o2<7mIYRe@uS>Q}1ls1gF2 zE?<-=+t>V~tS-DDV+ruS)XOCO)K)2Y=q@H4o9-Rz=sVg$eh91&P!wg<8TsYy;Zh8j zOq{+^?g84~7l*ebel*u06bc*93CK4U1rTX*zG(7rN1 z6PNS)GpuemE}yt$IVmeem?r}Lj~6y(mBWsY)yj*j3BA&4fjWjEa+!!+B$o7Sb&pPr zhm*S;2OkPeETr$<8X2AN{c)P-t!~!7APe00#*@dGkxFG+M)C14@$7^UHKEJ1sMYvLh?P!%4L$!WrKI#j4Uf#_-WPv?cT7$#3ue=$tfR~X?q);XMOHH34{l-c^cVBVl`*1Em)6ih76MK92KOTClM3x#DCU5A<)xo{p{qEKR}6(%v`xu36aS?RJkmonIOcbneC% zELSAl*t@v<+2VcI2w+8zL<7nhv-wH^z&#EI@vWUE-UW6F_}2DnN5FT^C{m#ArlWDY z_r(k8J1a${&{-z+CU#Ll(G}1A<4>kO$m`*&`TJ%2#E6sw&DwE70p?(@UG~x84%e4} z<{OOrqf6)W6es-L^4$mtRG6IWt8LS=7@ioafqTW67JVHZO>g7%zkFw9ud4GapX>GH zaz%87Ww>y3bZf)+TKio3*8fo5vGkmFjVvEe7f=goI=Mdg$_d;d>TQm%;Nrd&G(+zj z7$8jjDVCls?e{ARHr(e+Zmmbr>+?SDm))B;k-9TBTTD`;c4p-LOVcv}uHPU2RQ*&B zK{co8Mf<_UTPuCWe_68GQmqKL1*@UjfM`a|=>4TY#}NaN^%@Jh4~uJ5dT>5%S{(e_ zdwNkl{j&s_Pe>(MPTN@7$SycWM*er2YR@w|l-Kj72ivV`Q^zXhwvkShk3S2jhUi0p z{5+3FQPx!sfNS40FirEKLvrKI3uY)6c5Bw|c|1KS*qocN=TuKSBl+x=#l4-|3jXGo)+4cG(S$+(X@raseLKIT`JDX(>yV9N3XL>-BtdRMgFR z5p2-5wt>_evHjbUW(82EG4p6^A^m8Y|szy-scu;R4b@|joxOB zSi|+rFsW=z_KE*#R-c^VRXg;?O}NY?W?99o2mAG;Zkaqji(38MSNBqByG$Q@AY%bG zp7ynU%-!F3@s7P01ylWS*D*PtL(6aPSav*3U1-a&_df=?@COF8-+5yfxytm)i8oO< zi4+tjbPawW;ox&UxnW`LD^pO9KHkMsqpMa!ZJF8c-QR1Ah?WPBihLSEQA&WPB#>{t z>~;ZDJ{T8K-hAb!P-KfWo|w0tcr50hL6<>bn$ZpZ%@;(b<75=DYDkXJDo2FbpR#_Q zY*q$NT+ighwTbZYY*R`s8GUCNfs9}VbUW(r?qc_~EKufRAquy~69P!mGRIuGRNMhU zNra85e&58X0kBE?D|9>S>zn=M{9Ff~ ztZor!n+zP&Jk;^`M4y1;Qjymfum7~K)yt9#f&JNlR>8(^Gn3{d3VW4Bznjmy1(ZXn zcc2?T+~tgT=T4G8H2)di>M-2@YITt4z{c`!i%Rca6Npj_25mHSO>a4OpuJ_~i^uzC z@v7s$K9?JtelJ|o&dGY{M~Z}&kJo=dE3T%p2}q8ph*Oo6fYV-5cQ%hUM`FaJPj7r1 zOWE89U}!bCZ0-0Odr1YO`uF3!N`R1U91L;EJ7BqFvs4XV@`m)x$89kOi=C)h27j!t z;tNA7ZVNq8X0OcyP?f3M+vM^-ATrZQ7Fkv^^#)!?uyB=UwbrS^Wr?N0Q++8%{<1`v z6Rebs4g7n;dX0L$WzFs_VxmM`KmTaYEnqFBbIF zjAG^4-PYXvbyp%5zZwnn^YUT>AZ8DM?9gVGNXF-~FRJR^JKfYX{zP4A!)hEzzK3sP z^9jGmGBf|n$hFG>?aC5`>So>yY>CmsTHSkuKfdnhO*UwGY%dvwBrx6(hy}JGy0Pwj zfhisTeSf`C5TCC(rOZs>NzeprnKrxr{c*Ed=ud_kiXr8SULQlj?SCglOQoY$XWdH# zccAqw%DGk%9$4jz;aVIzMSk#=^inB?Pu-@dJ;ojAYkW+EX@S(?uUb=po6GK4s!K1U zNA}?H^@m^e%tj=)9gl~;Y()tL$oy>3uCw|5#QC%)(7y(U$PranUOQA~m-5EMqa54Ts&Yopm50V2_ zDM5{H4A7@dNtw68`u1UcxFy}`Ki@EApbhSn&#+hvD-;!j730Kjd#914@#0ZIXtrOD zyL&w^T=1T9tsLW9oXY3m9*DlKVmtR($rR%G{b7$8sIVCmObWOkzSyLgVEvZ#TF&^+ z?N3Z@IaZ8`iS|j_?XAl0iOX6qs$c!YLDths-*?B*^qKNEkujr0nJ(xw#j%!@tfgl& z%Vfw+99tuu^;x&Pm_LPl06!5>E21$zt+{eTv;&>~_`yZ+x7%U|B>CO@QKPdsRrM(G z+qJpw)G)eYc_0ibKCRC+`Sc^M@6hYkP!W4R)W>I8bS>9yBTY#GKhWAOXP~4|7TeS> z;N+9bFklIV$idH!2YFEB#wiRL7DY;->FrVd&1xio*hn67>Unplh56x0x4q(Poi=F4 z70eP)C9Zc5wV~gpK67OhwsHvhcFO{6y|_F#%(6GUwBv7V@?RM;Z!cUBF&bHc2gd@I+}8O zJq#;H8Bu>cW>at9E3yT#(sx@a8l`_<6!VcO5{Lte0ZxA8`t`o|%o>(-$N1X7hJ2BT zx=>43%Xj-jU=Hw+q&$#?+BHppC^ZoAoXX%he*7&dTV|kH@RZ%!RO2=Mq%tlAZ&*PNF>XQU?XX( zh4MSw`OR5u%>D2i_3nX=HGp%)#O(pj?`SkSHVgs6QYG|4M853IfxPbvcl5{kesHEy z)(qX9U;K3m+|^L*Eb1Uyr45o&%spNc5Tnvhw(q=(C@Lxw^FwbZgnd}H_Ivk@P%ZyE zHz|rM(5*ybw*lP|n*p>L$K>X9{<*P!z|nj5sHLn@^-k`Xa*TRpH8aIP3^A&mL&idc zI*BLqcw0a#iva_L#xuE@@8>qw&pMV4xp6ij7pI1+^sY)2-P%^-Gp?q#d;dPQJXyJ~ zjhq+3_e1bPgC-PnxxF9C*!Ax2z%?9ic4Q-Gbr!gu+IP7whE_fD%YP`pP&rN+zYe_L z2sHxFJ}QNyA;5}6+j65qB9dt>i9@_SrFqMejcv34M?rn+A_BCoCtiTi8_Xt_3#MPzcQNsh5+_?9|6jmpHFq=(3izf=?K=;_9`QHv2tD*H#e$EnSu%gfA(E zg$;)2^l2{k2Q~;CBN~5?9QkPfy-3-p47k5vv~O50$qqn(c7c^UsjD_`1a7fOdx55R z^+fo;IrO#Pz~P;=w5HManoGmLw=Yw=cMF>QbivrslR9jJmfu+8d2;haueTy8my;lrH9O|l|F$12fQj|nN7upPT=11 z?1zxf(YQBnPYZZPQ3pwDL#6q(66@KZI$5F*1$&n4mhqV*_#u5NaD12)Tg~IzcE`K* z0WQVw$FtF*LzV>Ga@bfo!@gQ!2S{415hImVjElWJI`?O#5gVnl`E9Xa*93d@81joF2WoJ z0?pWNdYCUS)`q)Ee5y_rk8wAe=nR$QKaX88txLWdPS^judkdUhdQ)zp5c}rmW zt9K1(lbQ|6rfeM_^Se8F6jn<%R-Mr2eL)c@DZcXS>04l^{=iU0!^{mX3C@=T zLohw_&ibRsjgg+YKAc_%>a}56a=88=EB=R zLwhJ9vUYIL?KbK6pW?*t-l<9flP3Pa8(+>? zMjGr?ItYj|Vsb~sn5Mi3y&M~>`onqQxi50Fe`9d2_w=ve*4yk@1W4ku=pQ`1^Si5y zYu{$1ujq~-o;DkFhS^%DpEt zf53ktd`TuTajF={a2N=|n_Bq7IjNS(i~y6mD+u`Z5=v;Q(~8SK_vLN>PGoBEiL0_d zDRFY|W!XG7lQcMJu!`E|;(NJCle7AJ5kILx#ZF2 z7b3FitzE8X_s{hcBX58|7CEpdM6&9kMmlVe0;!xYKcPTA&u{o+1Crt>7XOn)qsfZM z!82RPtV4|VlLe>guTA1o2VhAShY@XSfI->d|2=B?|FZ-xP$1LPtyB5@Fbm+^EJ#sS LRi;A9EcpKbrWH=+ literal 0 HcmV?d00001 diff --git a/webui/public/web-app-manifest-512x512.png b/webui/public/web-app-manifest-512x512.png new file mode 100644 index 0000000000000000000000000000000000000000..2687bf62caf5cf51b789e966953f7765c0041fe9 GIT binary patch literal 24080 zcmXtg1yoeu*Y=%(0i=->X;2W5?i@-`qyz+{rMnwp=q?eEt{;LR-60H}5(457(p^J` z2;b%Zt#>&~M&_P#_SyT{y~ikZRYd|^YFq#S2wp15zX1R!_!bJ_V1YkQy{E3hA6V~| z73G1u`#*W#i<1F>0eC4dqvf5ox8Rj!Xl45Jc2jb5-e~iW_;)XriGkELrnx4Qs_H1- zr^FwLoq1?I8O_t|2P{d22x>A_1`>)wDHtHKN=QtcUO1w@+J&3?C4XVki=U@PP92*! zw~6+{HAA2L4*iBs{`uc@{{A_#Ix!+S_Mr?Q??ocv4C!NyIEFT`)3>HnXW>lj1hl-6 z9!6s9P&H&E6ai_1V6yZ-W%}qK2j~Gj45<)i7zQ3zRUVuZxNc+Y!OWu(Ci~Z9JlU5M zaJ@}{%HM|%5Sj=Q5D>J(8D@3Vas4nqEp+i<;YS7_OFm`CMGi1p5-jBruROxI<}a;~ zs;{n=5RsC~`A9qRN*CbDFRHA3!LfwfJ6<^*)}#vjg?O89GBFiU4q#AWId>5EVMLqW z+zcc!n|F$APmYNH^GB-4q-}zD#Q}IwQ&aojnykfP#d-n1ImC%bNYWD!-uTExi^<1#OQC1b1}=b> zmNv!N=YS;q$O!QG-$vw-qNZE9l;d|D`T6-TQ8ur7Q(<}a#l?F3J=l z`IL#N=6M=WG2y=IaOBOzU)6dCa+cT7vt++!4+q{~ajTDc+5s8o5XU1S5&bE{7<)lP zFLlF=l4{z%6e@0-TcL23RUq!}7wxJVXMqt& z2ud$9GIDfFlkBYgNPhZf*UkKDO}BmOlNUfsOUn*{!e1JMzJ(Pd6H^N@#NU`ItWnpr z5y2#mEOI|Pows0{)m-Y%jf{Lf3K-?f(eCYRUwNSPxO(fD0T7f@#ymG9z+p;u7>JUx zP{!NNqUD8&MEpU! zj*K83CjG6vy!nUkE$Z~!TL*sSdkmX=lUzW3S?gvu@g^!Gel5O5S%^pjvXOxet!k_6#iDs00(Pp>mQhUYdhQ9U!PMAH4;$${r{ha z6vwumb0{WANpkUrPC>{2Oior(t+WCe<{J#)Vpa?hi1vJdyE8iR8~#sRxR=*mfZ^66 zHgQ@?P-LVMHO6rG`+3h(^m2r`zwfhFK+Mm#VkYd?4805jL-h$Ud3aN0YwKYVpVS8w z9^B7sk;8-yT+MZ{2}+wUJ&!Ba$!l+i@W3yODb^3-J#^(01u~e+4DK^M z!HgZKTcM`g03Bag@G9qur3Y|6?YccAA<>Iadc+2vK9H1~vR<3An(TglZUm#ffZ(L8 zo{)f_PnC}M4u?lz(?2_7W%q$tA8v2!5FEK-l(3zFD$i6Cq##Gm{p^gQ^CJZ!Cnsk% zi;<2D?)FR?RH39FvFlncCic$GEn2yG=B>ca`Tof!=(`v3{c#E=X|Nn#t%hBO?c7JbO9H>Uyu5 zbIJZx|}ujZaUyrI|)Lk*%76drpKs3U4~u8ei(zKV98x0^Gg4X0~NNP`AydMMjp; zYzUg-(@6Py+p^M|_iF+z>e56~H=tyOo+QomT6tZ|cxnMHo@WaLd3#<|q@Y%2AosId zuLIny=-+38y7Vx7MGTWFMf?FABx8=xX|49VVggV<=$H@16$k$&G+ja4SbwOY2;=gJ^ zl5J=;pJC`Nuw8FHVd8Ajo)8rgX=*|(Ruz!|PF#1=_&A^#Jdx|i$iuU)-J|Z+BjoOR z=jsuB(!gR^o#&OCUhz`|dnG(Yr~6>wD)5Kw+kWpi{kk=X@nB z(RL6mT?qYsEcB0kd~#AVQ}~kx679f2oz%@14bAN*svN`am?5g1!Ooo_IvYuBd)+?r zP-<3yTf7hN+XexRA-9S9cSPYXEcV+r)dHJ@@DmyNG^RJvNrzO_)EV<5m48lg;4(V5 z#KA;?1t%o3$lW8xB_gIJBIF?+a`l|TVW9o@>B=8WBQjGJ6DO_iKU*i<%CK(a;D7Pm zb}$-)!PV_`=nG+YGW=9%0VThGOTj-n#3BmZcd87oZ)}|Wjwc-A?fEcnfD7dJ*rwt* zr91=i3nYE$m!Uwb4!wgBKB-@I<+XDEDU;)aG;%zH>H_!BfN%qCer_qTJ zY&F9jN=I8v_;ad2KKY-EQ|(KRPup%@zE;LG|v z&xDFfZ|3Ot8@dFpjiXD~oJp)TbGjt%8)4vX`%MS>W2MRdK8AL58BbqxpeuFbT!@lL zv=Uo%o-_{Qx4+{OX&?2vSu?*_^s_|;o0@okNBrG9X$s^@V7BwAB24UN3CnS!qoZrz zlFe0)2TCe#{O}P(fi;ra)5N&p#Dp)v*OF_qg(CYS{t5^ig*MJo!9$fY@!u7}^cfcr=v!^=ndE^2Nt_nL z7LlH>ZXieO2$s}B-Pf?LhL6IC{#4gXls%nv_L!+ucX@=le(l$@>C~g6qp7ffP6b6^ z{tq$}pUVF-xxA#*b0eQ#PK3&HMD7ljb-5Tjxk+X~?zUX|8Sicyy}a}aBAqe3(!NbY z;T_G0QcjU3^Tj}n8EsJ-)B#jHe>6^&4;% zSun(eaTY%qxOT@lL@MASVT)MqUKfX;uTgsWh9o73iAlIhqizQ#o#Ad=kuo%9jSyXOP(FX}v6jwlRFl$U%K9}?i> zW4=M%uppeDxsi~^l-fqVU{8$H?mRpjjyLlSlO@b8f1FI){iaXI-QSBn#E-RBchDso z@lE!dJ!YFna)M;4#KmHk7bZjc;Nla|>zfAz!HGdaY_MVDnas zAaZgzTj_Tb^*3V*DNYk*hg(|72M?~%0_~m$9imPU;x6xh6i&EL*&i#gl|=SwJgN*I z8)jy22#Wz4e-z@*8?E;33p&PrBS)Vd;kQfwsE3*xh3B`?Z9W6IeTNziJoTnhMpZ); z?>-J%XN1dtqt-rRFWE_Ttohj}E)dr7Pgqmjm-=`9n9EF60VN};Sy>iwY*yYlPBQag&X zT_Clynn0B((Z-uR?Sc@v^FEjMKbm7>3?P>f&3w&3hEvF{5yEq^pC;b|TsFTfC^5iw z_KSPmINfx#BujYb%@?9bc~WCWBBOzRm!%09@xYunwiaQ4!3&k2^0FFQI@63xcmwh8 z?E)Xs%Y5R0_C&bN2&toLj{cX<>E7PRkkjv+tkPHFuZF*(Ow&o`glL}`_?=68%iK{y8(mIwhn|D< zR(v4|pl`JjwYaP`dP^$mut?|FXr;eb4^wY621 za!3A(G-(p^pUl)R0Z?%V<5#Q?ap}4(AQf6^`_)AfQoI>`{2T_HN=0d!3j4p8mIRzZ z=L*^JIA%&B^_GyT6ZFWdEsr1yBvqmmftq0bWf$=G3?!yU(ZWd^E!3l$J^mqU1MlXk zf&5526XzGv(Q4XSAV2fjjbNRmTd$ezs({a70Qx68Wksd?@?5f>zpFspIt>w7!OA3o z%X5Si)QTMF9wJGKtkaA}RSUESu*X0b zKWmUYSemW9r2|7%^i}XOIbe_^t3ZMuKA}F@Q+utlM|knz=9R)*PyCk1Y699o{*OH& zcn0)vh2$Sa$w~J)7!(KyG#9=^7c)7xN?O|;z^#;p>2IH=t#bjBb`RJt=Z1qXhl2py zwk)Z_0_QR#CwlZA6I`1DBVp|p^q5YBvsW!P z*EulD2-`WByu(BC^f)%V3=BbU8Ct}z)pZttu$UXpkQku6w+y$fxI2Q|@R4lbXFFT# zzxY&RlmK!G2XIJdBAc2PD5&+I^rs`d&{v7K8|M5fnYZGmgL%|x7;vebDl#hu!3W03 zfe8+Nd_6T0G8r`;=~7Zb;$Uvsw>^TuuP4sFjmQofoaOLWT#E(IcQF879v)&Pi<8}& z$wV;H!6Oii1GU~lF`ss4hV>}dPT0IZkLmK(ogz?jq!z<^GBysuU1GoHT3(Z!C>ScI zv1fa@-o5zLwFX0dqOG7WF0e%`G*g5cp2-X?OSe>@!o>p&Kb6{AJ6g;6xv?5>HSURM zv50a(>ry`JFL7f-FV%DDA#s5X?k=)h&!Z6&OyW4{I!H>mmp=D<{~0H7pAd1Oy{&U< z;dWPToHT`IO$66#ow&COR4=~oF_^0gYM*V{?*{d+CeenyVi+?78nkz{(SzS1AP=z# zvBrHK70B?CX*?6hPLG+}m(kHDvMt+b>RRk`SB3n}V8=XgJYVAfD{Fn~QWr65!9oUc zwol6;jp>`1oJc@Ovx(!T=;ovaipmkL;Q94YzLa!Z>7IZ&;VXlnpwK-kmbCs=1sy+b_SNZ{LiI=FBCsWq z#+jz03!TvMHLF2Z=6qmDw=zUQmc+gB;+V}Co81``nKX-0T#RY-u9qK!hU-5dSh+xC#)7UEJ-jI5!FnF-2)8@lgO!#6NOssSfGL3I;`Jc8>|f zIo0|2wv~N7u<#+hl z2^%!RucUFIUjAC_}wr! z@TzD}{P`KD`$ z^pj{5XtsQ>K38$v4$dYn(!)W?wi%7CPxPYubGa%6^Hxqk-ag)V!z;eRf%pLe)xF^$ zSb5dfs6`}1`nK+YOCz%Q9Lt5FbB3&SnFA5mQ{V#HGx+k3<1D@^Wc;a*c2EZ$&eEuu zBVz-p)H8b~l1f?44;*Ueg+Ep=a2n%fi%p(7j+75)0HZ>u(Ml-MI~+i`;eE zP~_iMfA7a*xK%I;Af`-pDzD;;h9`av3k5Kn(^&6wltcK3!PX;V52*VJgB0g0Pt+S*@1LNvpb=mAfi`wY!@#783w4b>)`}mKQCzilGJ^xZF&v@USS~ zqcYbtz*H)7Rx6|Q5UWu9O$!PCt&#zQYud0|TLwf)Ol24!y3EK@ z7N<_2u5*9m5sHPr7*J|>R~an5dm%@DW5g*;f#54EFI5yFOQH?I0o0%@ZcTh&mdo2@ zoAz{pA-0F{$F|0NUXa$5JnHacj8cRq8 z=#|&ID%~HkMF;=-LReDzR2+8#&zTZPk1{5|bPfdC`0^w!Ex1DZS|~0hq)@z(Mximl zp#5wJ`>nFDG52r~EU)MNgoxTT^W;!9koaIhZ?n^(UV^1$N*8!F0LrYo0z^Xr-p^({ z#Os}l)(_UlCnVahZgx`DmCYWLg%ukl`GDNv&ye4q~hhu8SFR zAz?_@b<*w4^s!MNL>G5pf-bj|bpB9nqj;DQ?*0J0BQNo?lI^o9fGiz!R5sQjOVz37 z5QztgBdX-c+|w(PB?I_GH9)2(=Ri7k#X8L9jEmsR8!cYJn5WitzuD$4uofA59%%l| z2L*`~+6H|UXtWE1W1?VrN&-K^`o!qYlz)BPBm?vYuv`AMi8V!Drf_}?&v;=?opAlc z4m>z9tY-mwc(_)9cy9_}3wCA;#wGlh3|sek$e=gkQ195r()ktf-URZSQ9)x|cX;x4 zAmg}=AdzzmV?pXtD|IIOJ>+*jsE`%ctivk~2y+ENBiNzssuoi9s5lTC+73LdW(Qf{ z5Z4L}lm?VfbYld2M#4-oLy4_zt=EI3&22!$CMrCGUM_J-gEbV$?WvQdw#D-{I=a^~ zzuxiyA$X9%0#Igu#(SRq-Um;^Y8;NE&N)*P>a+oBY8vaV)OXhl?Zq#(=fbaNE!l88 z+|7v$aFpI|Xd^B>T?IVFFOxeII!&l1oV7ry6fD1m*R*7o88q?rh5T-ahMC}jqp34f@^>Q7ecoa(ra-I{dq`%c&?yE&Q)dg{iEVjxaprQ=8K0 zj8b3lZqe~}zyQ}D)PY>&U#M`L^i`&oruNuyz(A@94DLzdhBrSsL<$v-ftip>(9x^i zFwYGgy@1yw8QY=r2FDcVEYW3k56%S7B#sj7V|1X&*-d0ihWqmrO^-N%G-5Q2NWO<9yRV7^R~zJ|0qG>g~*6JY36l9!tsdY*Z*7YJeYj%*{X(&9sC!1fJju+SG zpU#HCikP`X{VzsC4sQpeYf$)n&79!huFd$aR}#lz61Zcxl|~3tDxTBHkXx8x1TIEk zK@JySMBw}Y((#md8neI=Mnb8+{6>9OQ+-!keN$6?T_dlCKQXNB`;VBvn{G?KGnDTk zB8n_pop^x*Q{dijwpre6lf6l2dGnQw4|Uog^_C8~lkNbN`2d;TQCcwdL(TC=S`&0y z6C_#_MrUk^3}&1 zNU;iPb^@wTJ+fZ!TpsP6=&d*g1x1ESl2c=KvHNyWE8-3a9M~MK+=AtE-q&HUCC+^K zXSX{RL!q?Z*=yXlzus20S4YcsQtzHp)?nS5aYfvH7B|EV&OoN{5@4@T6ZXuP9M=#e zO8g@s@GGRIGu~iB*6ISIZ&FHg+7rTY z!E_ExKAhd^Gt=r5$vUR@dRK>OVBV=)^V7o~BtRP-y|36`i+%>|!A&<^T_fbrW9Xah z|Mo;IrfJ`RMxN40Z}-Lg4?g(1Dd7E2u8eAK+(MBTMuFhZ*wH7hECH3lev}9QTY064 zsSV+9*>KIOQZepQODLY#bgPT@>yROR;jcXh*&{35gyLn+#yEHYkhURfd;x?miIcF$ z1sfr6Rya2(Et7s&SoHH2QC9}f2sIsL_!0$E#{elWUI?Riu-{fEdfXHe*-B_!F`^jF z%U(_~QQ?ECCiLltC6AwEk`vdu?cT>wwrTV%&;bDmtm zaoRT|OOXJ1kPi9h*N>^*xt&L5t=f55w7v7htbu@KL|03ih!;71H!C#w-4hsg3Mn%| zfwW^w6~^C|NegZM!@ZgPIr_a~;<z=yVG8yj<)5e$2{@`CyAVXv(ox+XdXLzf$8YK$zR(o@q_6uG@@dW=C*R4 zH2q>t8U2}bM!l>rCI_}?i1Nx?n`ig8o)^bI2goNx$a<^quIf~F2LN;ufVWBJUlsB7x1Q5Ee6yXUM1dad z7e+@-2prSk=RNoKpv6NJUjBnzwgrH36xcrz0B_MXGEb&+k*V7$@?8n-kWjvC{?QJ@ z+O0*7-B)&00=q8>hKRYduBAw$bT&U_xH#1x084RB{@w%^KhOgfvsvQ12H;hfo1l4Y zxJ0-4%6#}3tD3AX?{BEwxp-=`v^tT2(w}=4@r#jW7+Y@~zq)i6iaGJ-t>VH#Bc|YT z{zt8)$zLiWX;qP@k%w6vcYj<*%>sSUKjU+RXQ-u4+9b8yrU8J>%xG&_PfWgQEwL!( z!icIQgEZQ(=(VVIiGaV$f;o@ilxIWlC;`#<=4;#XO-0V;E`5qs7%KR`rt~p)>|U z1t9ZU>)#fDQFcJ$Dq+;j(h--HwI582`Nj#B#)yj1MWhnLK}LHP)1zXfNM0DSsG0Z_ z8`S&c^z_`mwUMShDlN;q3v22?;f#;8A+If+v=rv|vlf8(XTXFKzO2Y5Q7V_i`yH~U z#cG4`G9UdNIYSZ%i@nPjBo)b_!rEe1WZ^Lqhhu=oBx_Xus6AjZtblgob(npTSirZ} zwEb&I{fWkgp~`N5@`*hgt3>)?f$lQ&(iyT-uz9`yhbF@P+QJ)l1=a`3d9BAknI(k{ z`k*FwTnR+eaoL&I^+~gDDbriNNV=9?T zf!o#7m(SgZaH({rNRnS}le#JVuo7wr%$Tk_JGeAsp>9pImiW%+e%nJb5-$#m4DBb!`nEVMBl7}bo7{^jEC%nBHH_ig-j^B-}PFD13kaj@5e$h`aQ5%$SNFZy5 z&6E97yIYqk46M0M7kGlR!Uj8kFo9y`yY)WX<1|3U=AJC1QkLvzR z8}RUjZHNVu<>?o-qqowt4Xmjrm%o*84-Z$);rySq1+!u!)F*P_R(4czPq-`vd5LCs zpdv!e8nbFFN#(g(y#W`2GO~F!MU*Mezw_YJPRgm7F844bzLe2)`1eGpymuWo@_Yo0 zxhO8CH6NWS2+Y6vVJO2lVn?mo*A>~&KQTf5!XaJJjBqc$7SPpp@Jq)<>V>MSx9+-5 z9kkrIKcWi2#WRaQy#8pB_nI#qUsWY15%j1lYD7*ZI&7g&r72;l-KC`~-U?Lt6_{f0 zx=O%E(QW@gr2Qr&`55}?vqP>@#$3$Mwb0i`;I?xgL~UESBPup|c`w;g)d*#tt>qn$ z=YQnE$1qz}^eN{iSNQ_m9Obh})pGZo0Z(e&Exo13Ddeo|F#X8g{PVdFqq}xBl16`@ zob@!o@>;6R^yltgDQE}?9JEu%wN{2}44C!+@oK1+WoBf<`BLtq088g0 z#2Ag*s-Xi|Sg|)#z=?g)jc(`}pEL{#wT+LsMa^df4~sKV)m#JTV)F<~4sWDX~waH5(RaqtCc-JJ~=g56GGb(8#wlM(4C+a(U@roRzhTV&*C;42ACk_IkiWk8)y)KoeU> z?u-iq?7e2fEBA-1igdj@J-pT}gkgtLMhq#c$KlfKncO)8OBU0Pbmac=vBV+|(HJV^ zTiYsmi)IOx38grvjp0WFNl#E3(W5!AJeh-?yZ4iA0d)qRlFq2JfufI}C6004vL?v! z<8n7`{Eh-OY%q!!^sF++Sm@0N&C7Xm8IkYNRxlni{4VQna4$7{zwZtH|Cy*+eS#s` z*&okZD6&0D8%x+2b^^FMAn9@1vGo4LkNH=fpFw3s4%e-1d?16LJ`n9NxZQ^f7hh_d zzV|&Be}!5HVnc)31m5P^>ENeb*AU*&L?lXW=aPc9 zo1cM33%q{*2A^2FH&ck$?5RBhib&39ctfC3=iXn%x-`Fq)ng1xiEWh2;#HwlL9J^-SZujR4nfcY4g#K+dRSsbE z-#}O%7g!%g6*6yq^SY*V)4ONr-pbkY8s^Wh@|c1S><$5dC2f<}=aS^V4c7bzc9oAl zr)P#%?A}*YEO4=;q*MD2J^orKqJ(Ing#p~Z>-2MkXfF4Rh_l4wh7)M4a>%?;RV`92wJ_fji~ZKdxyyR*zTo^BpB(rko+W zer%wy92c&j{eAVK*GW-CBLPj|1p(e#s+nG+%?7@WbUJkH#?+$lDbhB;n8Y{e@U?}D zTG`B}LZDwzn4*-~daqpGEQKhp21D;tDNda*Ca<@>w&$k;-bIcZLiy4i3&pB7C2eLV z?P6o5R4Y}}je+}JyB$96#%xkpS|~F3CbdEQ_qDJNcLn+h|I_3mvMggR0mV`{lt73I zvM;F3fg$Ta+@MJ8pS)X$v_9MBtKyN0-d^t4(A5vt|0y6N!)=n|c4z+PLe=sp^(}XB zMpkb<>AmADDA|tBFM5igc=l|sypWMiTvh{B1~_BpnXeG+N9Kx}y4+6TScc~!+S~DU z9u0%}4vqUl0Htf9r2QJfym(TtlVF;Y%)+O_*S3GeHc^|0jb&yRc_hMkc&y=uxt^5BoSElQ`)OD9;NGgat59k*B5D0F|7 zr>Z`u64PfcsXuTu8s1*PL5RX+$(_Z~m~hmJieaR%2L!N;CwPCbH3_$+eU~(h1oP0E zW_u0kryA8N#U3}Vj1d@7O!2V~OeQvazgiPdoa5FBhIKZIxylD~X}ysAtUo#s|Awpq zj1FIo-VMK9_!Y6!Acf?q67a!yQ_PidbRe-MA*5Mp53GrHQnBzd0t8@XM+_7X8GAA~ z)b*BSEL>J52t5Qab%BwmBQV4UBPRRKoQBJW?#Ag3lVx}g+|`k$>-RO?tbtTSN`1=v zVixwO+#l;ziPI!rP7ca>k={uaQ6`0ap}KNZ z7#VsTpMMpd387MYKKg!piWuxmAlcT0&;C-TQJSFF=xZW_A^w445yAq7?|JhSLrMr4 z^~pmuD1*4yg6I)f38bzK8ySc_90jvXC~>s*a6CMXWhq9xo3acG3!jDZOSPB>K1XuD zslnRYj-vH2K1-I_=D(IT++2Vxz$^Xfng>^qq$COm1-$8U>+7awiwjU+089=MIm>6V z$lQ`JX^ohSTf0PqoZi_fs<*5!GzO%kJ5mS#OM1n+zRofmE>{N2dbMr+%~?(>jKNlp zXQxozzjy}g_eR)E&<=+l@LIv6dHH1SumG!{7wJgXrv#M zIya}*ae|Fp@I6;2_vkKqHn3hE$cnfktoRcSZ(3=l=c{Srx5#U`V0WLtvHP5!{V5w9 z`fR_vd}y3zucU!#`~JL1a)JWJKotdE|4HJkZtuz-*e;TvJMQ^i>d+7~fj6x^yE^hn zp9}$KVHAg^Ag8wSz&gpCB^8o8XBqldyfkLcZ2QY|T!i$)WZIxtCdZeID3VCewaySu zz*B3C1USJ77loM!hZ`YBq8PzCnde%5NDSMS!H})6#{&vr5=UMS*t0aPX2PO27%ro(HEQj~1?y=-#@T{^#JB-8f9XRofS;9~JJ{u33*fKsB$0FAg=z%h1&w)Ku;bAw4n}|J&%e9t094+$+ z01x4A1eSL(*=veWF8BM|nrAV;^CW%n?v9;ym=X}iL+pQZ#zfd#y1$pZf6RK|S{%Y@ z3`k@_tFqU@O!|J!C7B|y_q6L7aHR|E_42rX)Ja@$k(l#z6o9y%vEC=J0Lb!%!n>|M8?)_~bYR~x4DRM< zu9(L(tGm8$W3HAA^2YPwsVgPIofA0!xP}9urg|iZFltDp6@fKP!QDn4Un8+W49$U@ z)L%_lNL2~PVQDD`bwF#kvgf}|i#PM!D(7`IIiQ522i9wDRwu7vaE)?le7UjV|C)?5 z_H5E&$hY&Q>-4#vYyf!03`l^50*C9x$Ngii!o1^Q1aw2*jE~&oh8j z9Rvo4Zw~iIbG2p(qNWL6nysm1ldCxZsNTM1ydWlNhrS*FaGjWRiz!+G|sqU^IqyYHH8^Ka2RO4kR2F*fE zau&g;N0kg@i42kL?SHQkub8OUa1hz%cr=j2_!wZnw;QDTtM$Nh5Zoli!0SBZu$g@? zDb5gBnte#l2{e2htO1nLHqWXX9^&_dUXDGgrAZNlQi1vG_2d|cb@p55*PfUKA4|$k z!FsMjT~5R8R_!_-T&NaQRB{cTp?kK>ir^u#?VT~@)>kj7F~P`&itfsEq%0W}ey04w7l(4(1w^IDLc zD-c=%?Cc;IjKm(_YkfKlr0XLL=?I;`{GS%fD`-4A%M%*a=i0aQiG#sgUi*d!Lub_s z+Eu`-7Bqe1zc&VYLX~FE2cZnPrVQBz$=Sit-)iy=??3-NNsF`bhxb>|UC;%RxA2<# zKQQ*f1+Cs$iUNwHWFWMl;7gG{*f^kVZjmNTVFhMK*Zg0B0acX(LDKYvVy}>+6o1H%#;?RkPw!LU}kPb}u1?1`b7oM^A=d!K2 zDIIYF`nCtnU`eXL!3QM;nu83(QKqyRa3Bw)9sqQWGw!W2gKiRB!J`l~ulhe{C|<~s zH&)M1Tz-=bzk@s`M@;wHfP2el_??3|s(aFCqyd65WgyDt@4PhV8><>bKpIk%l<%gA zquD`XzgU+d%XMbJ%=k$c)I^r!L=!|n)Fsq(s__C$k?Nia8yQ8(_kboeWnc>v53$WT zrV7g7$y5O99y|_1WtEj_AXSktWavjb<$*8Bph1h8YB*&g-pFS+cNBEtuGlFDbLCg& zKF?#CE&S{unD$x#VB!;B2zWjrgJt`H>qE8*0#+^J)V2W^~#-3)glNN#v*#pYm&r>DEWAtF$m?>1waE=i70DSXWGB8*Ve_+cZnCO80QLV?&6IJ>uOU4VWf3&QE|mO!<8@ zNbq0nxo?O~8Th>yWldJlT0g{-UopUge>vKDe0i|=fG`eq*d;ngj_{!X6t~3DxFs3N zp7^dJ3HrVI0YwU+Qw!-_@btvZ<6q0FNmExfxz*9|Q0J zj34Wr7Q$_>z++49ZpvuUzTV1Uv5~(9Y6y8BveU%D-(M9wsTpdhR* zW)9Xt4;lA+yG{jk-yu}+kTf$8k#J(%3jf~63}o%3g~#Iput4VvF0=&urnjy6OV7}c zRThxXxe|okRWku9EmhXqlD-wb&G)@@r}pv(v}ZCX-jdhHFbwR3@}YQ*hYg&-l|lFg zROI!YU?U@0St?kis3`iC;OD)SP>7$cTe|>(kJVT}00fG(8-(YhuNinc3+MX8T@8yr zS<(w|ZYDDQmVaGJZm=Zaqueg(t6U(TMF*iv0x1OhTawOY_mcxoJ{J67_IY1v^*cou z+?NaHpgev%Y^SnDAa`mPX8y!`8^(Z-^0&@0dCz5ot#b8rixaV&s^$CR9Uuc%lb=^T z;r%4h(HLHBk4j^TYc*UhCD?EG_CKhTbN%(oU=2@A8YKgn`Dl&;WTsevk|*Xz2D0*= z>BjFe;1mLc5h3mCMnqle4^CHL644kW(&qnXZwNe1AOKy(M}f0F#f#fC5a|>b;rbcQ z!zuHID_TTS-B1%fj+11iiszurKNYQM^hYzlPO4AG?#z zEXM=zF(&&(ahJ4YHC%pVI|WH=F^c2ry;d2CA?k2LoZJ$g<7IaI#ojXlkTyW^39PGW zx&APGct3?UjFV|;2HkkL7^3_8i8$`d9;gY%q?sI#``?O{|CD3iPOr}DbP&i^|7#TA zPb|f`IrKW*(C8pc`?2eNS@nSk@J-z}CUHwABZ4taHJmRE=?BVr;`8x>x4zk@GnfXa zPQzG0=0E&s^Ea2->@TP+tKDi~0P%EO$oV-}>(MlrOTYYnQBUhV?NJZDe{i!mD2gH( z>Usn-S$p<}PzgZflsU?PC;>J2>B+fH{fbJXAjp)b>f;1Ar$;_0-k~lM7(;z~Y%>@l z@(!?gcB~QZtvrIb_dRyB=@B>jVHU7m#~QA~)A2EhwvZG2t`e{WmWDx>Oxk=?)G-ti zjN{Z)L~o(U7(KI7s1JJ)D!f>~MD01OW` zgutpj;Ox2uhQkA)zLhO75Y|{Y3ZNujz1GIO-TLv}0Ij%r)*X?3azO8_3lR}KW%0ro z_o*i^@VoAT6FGgT<;Yd+Ra|V1(Eqp92CCHE5f~&mSbVhmO!xS{KTz6GkIfy}eRzBT zJT8s$ISP}WPq4#U0rB@zQlJTu;<1T;he5~zQgO2+SHqkbrNF-HGQ4mX*N8JaaBHY> z(@k$UxcXtcp#3l2=C|9x)Ky$$4_Xt{1yDXwS5VWo)0xF#hhPd;OtgNZ&@$K*XioN z%EPskcK4TyckB2+^)?NuH=96wDCV{pgZ;S3jhuf?C*K3Z<=qPR*y>)UIQ+O&3%vGh z+Z&<=yDCfv{{oZ2mQj8mX$t*E+JgBvyByYAGze}gaFCA^9z#CvxlczG9)GpU3h7Ry zjT)6>!}VNrWhQJ}uGgJWy|Wj#J@5FL;%Rv~B_`!@2o~J_b2f(nszgV>>hHav<>nat zK@lTaP)3F4lzW4j;~op&1mE$54vVG$kUu&q~EuO!Zb3*Tc@vx!ywA;N*3n6a8T3&7d) za{^`p_cM(WOm61)Gogj)Z0Fuz@^N$SWVlmDcP>}QXk~>qM-GYlDG;)Z2}<8^CrYvI z+e+?|MCD=x`E8ew$`W>4vAOZ4R!U`3O(m{MB_*Yla+Ri|eXl~FLx61kW;p%0D-0Ej zElFoJf2gW|@*TVQs5`>Syg#w88U_GAzY5k788C}urN=^OZ7p!SJdU(G|ELr1>m15e z@x}KhlMWQ|^8n62d~B`?O%0A(#H$Z0bh;>8PXF9mkjUz?XcReqVx-j@G!F2=;g1<@ zy1wuiKtaqi=%vGZ(~~2$LGLST`A|u==_K>I2plci@1Rk^_(0WmXeDwp`B!5=Y*(P# zRbzqc^p88OudyeFz{Rfy%UhE|ytrBRu2-e<+XJ@Ys^!aHNvsn7i27oJp#)XQAyr~_ z@M|W!Z=KqDzmHqJ6|^-br{mAd&hrz;%xvCA?_xyDYiqk3c?ni@Yzj||FJdSx^tI3` z^d@Q3pJQ2y%E|!UORe~s;ALDnt}dj~`K%jZ4b8d4N`Gso{J zDsPb7G(PvbAPvpig*D9^<@`HvzB0hdxuywyn#rrA8Ve+Wz|+>HYP+8G0g4hjAB z=1^t%(g#7cuEMHBQ4OKW)p#Na#1(n!eG?vlOQk6=P;b76N4!&sXa2p!vAwBoNX@RF zIq{)eP)R^IKXKUfLqO-zE+(0_!fKAx-iI@e>brWcyqfYyrEh2J2c!FASo=?!*f;iI zCaNBXW2{FQ*iQe%GYh>SS^H@2z|e*TY=%(gN=4^y(?u`kuVpR%`pe=>oq6V0G~#`& z_lT9eW>J;66`)WRvojL=Gml6e#vZS^bM=;6Hx5iLJ3w~OX7-oOHVKZu#K*YYNl+b( zH>KtTtiC-THQBjjoT@+aHYrvr=Vl(?_}IWcPl;Qw`DJ(9ZPG(G*SIT+HMeh&mGHfb zzwDcWhYz7g;bH$?oiS4-46259cMgv?UgFjDXG!9&vPcKI()F|IUFY-)0#;ug z54w~C|9Rbq6kyBAF3c=a=!vyK?v-`3Vd0y^AqsIhNpmL2GQ@DqsK2uAP6MO=m-2NE5%P=HZVkEIo1{>HHw zikE9}+=DY-?ft>!gArQse>%3)S$=6Bl6XpIGm@-rWVQA23JB^cB?@Lcykd5IzX!i%q2vf$7Z!in~NhwQYjar$xJWwW`)XmB0J6;n= zeOg>C)d(1JkNIFk7cgj%E7mn&k$cofPE9ksGY&CPsN>A-_}RUOwPhNtBaE zpM#)x3$3}_%C z3E`uX_cvz*Z;rezzo=u!LJXGwPaVB)l5xY zU;nSzlujabvmIQQYIJjw^ph4Y?@B|bzdC*gPE8pgJF#!UWR<;O>Y9&f!4!-**5dom zP2C5jCLd$^?-?g_EpIv)65}CZuj#}io1@IYn39C5B`8)1FCmw-&E177m(n1;cv^+62%^*#Y z>7;U>MMas!vG$RmK-sbHCun2zoaSR9`kG%34be%pMJgts$8%#$Ax)vh0p}{t(7_Ba znUN5Lu}N&iH7IG2-{>yJrF`0BsiO^t_#Z#c>iSOB&S%uswML~HCjqj?q5J178&uoi zQgLpzEOL30d0z0n0vH#C6f63)tQJ)P_FbCNe<@C(hlGtEueJ+Ib~a;#_hV;gW1R5a z?DA69j>V*I2}b`pVSxj7_bRz|lnX|0nVZ-Z6 zc!CPoU~;;RT_qt7!ov``vm(HLaX&yURR6Pzl1Qzz#%vr$BibD9B#msPX$S_V3-Jy6 zSEa97_Nd(H!u%=TSztvEIoYriXiuWM_Z+`jftnY4JEjw!znk?1h1>qlabhIOG>bX_ z{O*@n|H=+q@U+DDxL3pU39FiN652oQ11{0{bo3kf()zv7DuRT&tzEXY8t~LdC;a{D zKb{9_J$+>--7Ywy@WnC-i~;5<>A^2OeFVv7Y;Fh*>A1|Nx} zEQ*(=C~-_iUO2DC$B<*N$d)=FaUm{_b1^J+a!QsuwZQuN*So`Rkr(}7@-X}9s^z--~ks zPk4k~V!dEwgEw6~U1uOIzgE<=Rx~3-|6d7L9T(N}wJ%FcEx0sGih_WEq=0~QH%h0p zlt@Uq^wPD2bcm$VEx43)NOwpIQcB12-tX_df8EdLPMm$_JZENS&ODC##JRVP>k>!B za$vUe>!YAU%fMiK{XgB~Acp&EU@o${!XVHt4kiP2jpxuAc5Jr`JQNRit_Dy@%bklJ z?{n7I+prAOq0WaOcgAe(sjK4?#eH3(wr_9L)(DClP0*H$&wP|7Kj zB{PZVg#&Sxglxsr$FCOuz5+AwZ4yT9-0wsrW4ULrO;GvN7C5i+-#5EO{Q-|^j$ECd zD{YJB2Jn~nOGq9a7blMVEy#UqfyI$rUt}n8vj2Vi5}@W6kqTA1zjB-W{Ib0KxW9`K zoM%;MNN(aa$$m|8uW+Og5lPdk$1ry1<9}cl59j0=&6N>qyh$AX&Ew~`;?C&kDVckF z=Dv7$^6d{zTqi`!c69O1R4tYiwDfVa+P!5wtPT);4;cc9U+oOFdo{`FgS`0E$8)an zAR$D-pWt=nkNDnu^)&OK_T|K*)p%UcX&w`~ipGp)*@F%*_<2kBVs!<+5%_`70C)gy z&Nba-<|?qHrNHG|ntxmBolX9ihWxVqGpYY?=xDQ&%{`yH-kdGU?@3jFQjMT^oVout zQJfhp-Naa~LIe!IiLi;OYd+ESMF-w40JCvOX1xL1mVIrBm0<6Gp)BF?U&Kx5jl(cQ*V10$Ahl~kxx6nGV+EgY$fceUcb2PThLehNDhfrqA$k+}<;;>wVvia(_Ii&iY&=m3N`*8k9xuGQ((hIB18-EI*Jg6Ne7>uD z#xxzbC}?@{&)h}=%-^mk{_LvfPNTcLcrp9^8dLlPVA--*eU(S9xz7cc5yk+~Xtgsv9 z{grI4hfA-QS%u^CHwlUP-1#p-UrXQnA1+&`9~b{tB_l-QN<)rEeU8tjhdCD*cQSSq z4DWkivx#L^=u=5J3zf%_-^W}MU=CjO@*fbE#K{zc!($<_9x~gu$+D9;S(bELYqn82 zs0$C_$d5&t$HnzeI+748J_x+o6Q0Gl)lV^MG8qX~yePOpvOE2xS9G_CJURKCGS7%2o zKO6=%a#VMs=+XI<2k~I*D&bLEE=AyDcF+@*WTkO0wmWDMICfGucw%S!T5xOS&iwZ@ zgI!}W{ulRo!l4j+0p4&w4bP;a_Zvl$67AN~p2RHx6`-7Vy1!oq_c!&7k^eVx!rF-9 z`5>oiIWpSI)>f|9YfxwBO|rJ;+*W2i1_2pB(d*@^TE-Kybia+mcr;b2V3TiZa*eEk zdy1dx&Gj<>;=hO|Q~dTpAh(D(-F>!ecbug1#QJeB?b`Z`(+b7#v{h&xw=pbz?KW3f zX!j=m?z5Qscv#gT2v?dSQKvWPRj%63M}X54hvdyATNG7cf2iH_J&G2gqscDwnadN+ zhulEYu%7_d^WL*EjkhnqMry#m+6*#j3))mxbaw4W&e2_T=4eaaB*PYf~hu2f+T+3?4oP0vNt~UdK0f66pLLRX@c2XNm-|>8Vsrk z*46Gh?B-3}GHYm&pvMa=MlZe6xS(O)lU2+%90c&GJ0UY}lY%caJyXHPSs)a7_Z@8t z0@>QceXveZ#3uWKF#PJrmiXVF)Lii#0`&xVP=I$6FCvu-=IfSxflci2;7YK5gtu9!%E+qQr6wKeQT6 zjerv3u0PE|iEW+mzuqd^qTP~Tg*by+;(Xho> zAkL=Hp2bLibU%&2vNB9~OGcn7{DBTT${cIyS*Sk0Bgwm%pybsSG)n7Dv zNBtf3sowkT$3Hzme(I;#$%2G15r6zqvXY6U(y*sH>r>^U44FoUyd-PCbXDX#Ae6h? zLP0?@tK9eyOU8`B>-O&&NzZ)XedTIHrv1oq^_P|){&MH5U((Lv)<&-5EWMs!vhiS2 zARg~23ZL&M=qO&6`C5E*4$g@TM46Zrd|FCCj%aXk^w{6gWTei~mF!WFx$O5;=Ir5l zpQ#6c%&hrj?3q&YZ*@yh0pyRyxGRNlfBfuNv1%kGD07vI&c6X+Oj6MhZwM6eI45 zXC$;9TZTM`$-Gy;#hv+Q*QT){vQ@Pi&p|j;S%NT;cFIGc?^lwnQpYbq34S<$F^&e3 zJ`OY==_KkzpsE-bBcB@!v(coBZ_k`4(e&kpC4yl?`GtVJkE)A=t5e=6(+^X!hBeUd z$1WSpjgq9bLeMe0%%4W=HO0L!2U2{S64%jEPc*6jmd*ZEM5zsh$@?^2|55c?lC8bi z9bMVpl!O{}GjBHI<0853<$t zsXpwtTl+(+1rMW0N$3Lo@9$ITKz&at6*Ww$!%4dzZ0m-Sqax}ch)oU_(1veZb^Y@Z z|N6V~I2l5}=h4nyL9G>2BH3 zoTmg!V;sURi|OlR;dc5F`5P7k!yPt_OQAxq-sr%?kjoR^CAf-aK9v93JzRI?-_iXZ z8~{W? z(l|YC)t1aFX!g_fw}so;-Ymp;sET%ECN6nTB8uYqHJSCfwWg)xVyPG@JKP<;{#Mcf z2Vjw-|Mbnw^MT5fH7$Z(GhdPro-Y@D=KhTGQ~G6We0pBBIZrlzKBJzSPy2;!nr4jYv5^lA;Sf2toolqbZ_2d3E29^9WJ36}T7kqv6T z!C)}xYgd;kAz^rEcgka3ZAWq$i_^`_s^8P2=x?mAJQ)f;@gHW&UXO_UthbYTXZi{r zp@T#$&e$}XVgoL81ckd3L zBc|c4;8h8|-h9}Du(OfUyAOBHUkZ|HC}mbv9a*&>Pq2}ziCyve)RAm`qwM42wOL^P z>F^0fBQYJ<$mDnLRxyr^g0wtNj#EWtaot^9Jm`EGJaL>F9;T!vfA;waRZ(F(#h_KAfE{v(n1rc1Ftl z{Tpl!SK>8NC)de>AR^}jxuUr*N$tV?vUM%7#QFP-jUF?l=7AP?=``*OiW)fU$98kJwVg0H?uhKwgZB;;wQBJOfRjmm`vVQ zT(BG*E3$$ZpDG7ecSgc?E6f$G%bjkDkC>a+=s4Gt*DR{L*3zX;J7_DdW`qP?@Vo8S zGoyFZHdBS{zhSIP-jvzR0PK{U>oE;DZ zD%0)6vh!yIC3}y-d3Aa9*)EsG(1vXas)==t4==91qs2qy^TUVwy)3@D&|l30lGjK0 zRMG63Gy;!+*FEf%KOr`oTX_d91>33viU5!J#UNlYRF>=hs|qK;SejVt3cE_Sqv>7zDRERvreYH8d<~euvG?)JD-mtzQ}^ z%W3auMG=5+Jl1vx4fY6}az|1rsa?LUd8`=E@KJpI(RRzHuhqXQ+G@n?#&bWL`#A77lGqY{_#m1E;Yib9Go?2u zH7@yXHV67Fcg^8;+Y(+Xod}Tk%zRsk2idwKJ{dgpMxZQ1lL|W}Hj7s26_fu?W2clr zp%{g>o-zxQB!X(CzJQkVn8;@O8NnT}I$zM6SFLl{RF6@1s`rNH7?x zE8bMdYOrKl=yd!clG?Sx}Cje|bq{E{FQ_=_cYpW8r(&As00_kdc`g9Zdd= zfDtO^@UZa{l=75HuhLbfr=^~cX0+8xfly{w%;Z*<#-Y*jTRHy0nl@vn>~GrX$fX)P>%8-OO70&0VC@+e3QEiH$Meq_y*5Qk|i zUXjyF7J-Edv6n^!qxfrdG~KTc0S)3Ug|#=W3cTbY zQ9mbIZ50gBxxq%5^Z=V(jS`GC{V+P*B^uEZwkf*)J#igZH zPw|-nLoSy2eud#1oAptg`fB$q4)~T%mnkjU9qWUn>Z2BRbh@5jctrJg6PFr1x^u9< z(!{!Pb7)2q0N`}ZQNgWfCjad3ph|T?1rC==l;wyy77Nv>;?v(soctm}v4&~*6ZbXL z#7)x~bJ3AM2^kAj>(pchT3f}Or3{bGsXhMb(+k5gWD=hM?9mzua@$3;#C<%E7aBhK zimYYe|BZrvB|fc{g$42YF(RtlQ4&?_b?;CZ*^@IdR-<;%yVaVX^$m=KbBfS^d^X2t zW7maBCed{veOgnafw3D1O|=aD?9j)TEdlMfcq$`6$HF`!`hatXOj<^UEQ0(a88#sE z4{N7|^f`?J2aOyDRGI^dzNsL?T44xoe>QaOPjGcZFf?o#INhuJ7YQSWeps?O^m*q# z!=2MHKIX3{?PAbb4~h@M)G2yf#J{9r8%&zHZB*~^%Z(TD%{MOWwE{ z0dRD%7aXiput_nJ@Zx&|+^iQMzoFuJW-Bp!P0$;v- zX-6hBVp(-uLqXBUpAaP<&k(Qcqa(M;vxT@_%gr`n<3>~yHbAxqs(eO~atdCiJ%_{N z?3V=}9|Bn;>t0?|DX0t_H`BU9FxwR6f^#(Zq;)@^W>MrJu_uYTG1ekQ;WgND5ms~k zg3C24@O<7oZkGb>z>1msT?vRn|NpBX;rrRFkM8(ZM@RTpBG>>>Wx!p;%H|>UVEV^f z2GHzD)~ikhcTaOazaioI=qX|VK(D!p7Cm&qT8-Qk1;>iBoBiGXX9LNbAR3S#t2;YY zoy-_wie-a%dC;zn&qAe3v*7Y8{#2Tu7Xb2F3)j=1)Ma7~{jin*n|~mP&muK5Hy0?% z7}8S&00>6EYe~X=FW}!XT{05Mp^Oun(CD@B#_yMq#Ud!z>^8gLtzw^xWd{M#*7%q_ z=CWTi=hfpemhOd>r5X_MuU;p)66n+A2USG1t)RLY-NV`0@Wbwa4IeU}f%)+8K|%XU zEDr7eNv~33WhKzL{An;EE2EM6stOeERWHOVg}WIH6)qNNa=9!z0A5i)@@=zh9Bi&{svANk7ZW_r{RnErMzIod-Zi&QTxEZ(uFl)# zlZHsz@6pk_hCj&(mJZtG23ykR;|aa-O&dN7-eoRU{V&r5)GXd*8*2zwg@8CuzAkqe zb2~EGh@uFxB}-6O(-xO zjt=&iI=ss!oX2|qQ|skhL!m|BXL0*w$yxC=0RYv2<>7_y7_;z;U;0!3)BM$JXAq7e ziNqY?Ct2XFzDxB~H-|yU01n;%a*_Ab0sxQ}k2;JC>yd%0opQI&<#}n5^^3!hi-kJUEf(IkuD>xW9XN}KEk4kF47ZteWP`6Uqd=We;YIl z1;sgP((cQhd(*%_Wm&*pbPu7+ofXU;?8(T=s-hwZ+_9Y;Et#5{vXyno z8LLnM#go(1Tl7!W%p=AQAg@#Y8+#P&A?{HE-fdv((!opk!P80NW$zuo0;2va#+uR%W6n*X z6Be(SE-ahq)}5yWiuKIA480B(7kP9X%^@4HK#iUqNGTs^p!5n$k&p0d=CY)O#GhB$ zN?@iPpkY^S@Yka68!mcZW)%-;E)xKu(Hn;R@&rCizV`96v)hi@l`g9WE)q2Tf8GD_ zQ+rzh6^%B)j7r93MmE23!qW5u|Aj>>qTWd)H8Q48QLu0Dfg1_J zsbdquS|E3y3fjxQA+BRpNlfp3ep07VMw-%R0C11*PLYNl{AUoz@udnt=I=(%)da8w_yB9EW2Oi entry.isDirectory() && entry.name.startsWith("govoplan-")) + .map((entry) => path.join(workspaceRoot, entry.name, "webui", "src")) + .filter((sourceRoot) => fs.existsSync(sourceRoot)); + +const generatedCatalogs = sourceRoots + .map((sourceRoot) => path.join(sourceRoot, "i18n", "generatedTranslations.ts")) + .filter((file) => fs.existsSync(file)); + +function scanStructuralSourcePositions(roots) { + const files = roots.flatMap((root) => rgFiles(root).filter((file) => /\.(tsx?|jsx?)$/.test(file) && !file.endsWith("/i18n/generatedTranslations.ts"))); + const findings = []; + for (const file of files) { + const source = ts.createSourceFile(file, fs.readFileSync(file, "utf8"), ts.ScriptTarget.Latest, true, file.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS); + + function report(node, label) { + const { line, character } = source.getLineAndCharacterOfPosition(node.getStart(source)); + findings.push(`${file}:${line + 1}:${character + 1} ${label} contains an i18n marker`); + } + + function visit(node) { + if (isStringLiteralLike(node) && node.text.includes("i18n:") && isStructuralStringPosition(node, file) && !isAllowedStructuralStringPosition(node, file)) { + report(node, "structural string"); + } + if (ts.isJsxAttribute(node)) { + const name = nodeName(node.name); + if ((isStructuralName(name) || isStructuralJsxValue(node)) && hasI18nLiteral(node.initializer)) report(node, `JSX ${name}`); + } + if (ts.isPropertyAssignment(node)) { + const name = nodeName(node.name); + if ((isStructuralName(name) || isAlgorithmNameProperty(node) || isListOptionValueProperty(node)) && hasI18nLiteral(node.initializer)) report(node, `object ${name}`); + if (hasI18nLiteral(node.name) && !isAllowedStructuralStringPosition(node.name, file)) report(node, "object key"); + } + if (ts.isShorthandPropertyAssignment(node) && node.name.text.includes("i18n:")) report(node, "object key"); + if (ts.isVariableDeclaration(node)) { + const name = nodeName(node.name); + if (isStructuralVariableName(name) && name !== "LEGACY_STORAGE_KEYS" && hasI18nLiteral(node.initializer)) report(node, `var ${name}`); + } + if (ts.isCallExpression(node) && isStructuralCall(node) && hasI18nLiteral(node.arguments[0])) { + report(node.arguments[0], `call ${node.expression.getText(source)}`); + } + if (ts.isCallExpression(node)) { + node.arguments.forEach((argument, index) => { + if (isStructuralCallArgument(node, index) && hasI18nLiteral(argument)) { + report(argument, `call ${node.expression.getText(source)} argument ${index + 1}`); + } + }); + } + if (ts.isNewExpression(node) && node.expression.getText(source) === "CustomEvent" && hasI18nLiteral(node.arguments?.[0])) { + report(node.arguments[0], "new CustomEvent"); + } + if (ts.isArrayLiteralExpression(node) && hasI18nLiteral(node) && isLookupArray(node)) { + report(node, "lookup array"); + } + ts.forEachChild(node, visit); + } + + visit(source); + } + return findings; +} + +function scanGeneratedCatalogs(files) { + const findings = []; + for (const file of files) { + const translations = loadGeneratedTranslations(file); + for (const [language, dictionary] of Object.entries(translations)) { + for (const [key, value] of Object.entries(dictionary)) { + if (looksStructuralCatalogValue(String(value), key)) { + findings.push(`${file} ${language} ${key} has structural value ${JSON.stringify(value)}`); + } + } + } + } + return findings; +} + +function rgFiles(root) { + try { + const output = execFileSync("rg", ["--files", root], { encoding: "utf8" }).trim(); + return output ? output.split("\n") : []; + } catch { + return []; + } +} + +function nodeName(name) { + if (!name) return ""; + if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) return name.text; + return name.getText(); +} + +function isStructuralName(name) { + if (!name) return false; + if (name === "aria-label" || name === "alt" || name === "placeholder" || name === "title") return false; + if (name.startsWith("data-")) return true; + return structuralNames.has(name) || + name.endsWith("ClassName") || + /(?:^|_)(?:id|ids|key|keys|slug|code|path|url|endpoint|route|scope|permission|capability|module|event|storage)(?:_|$)/i.test(name) || + /(?:Id|IDs|Key|Keys|Slug|Code|Path|Url|URL|Endpoint|Route|Scope|Permission|Capability|Module|Event|Storage)$/.test(name); +} + +function hasI18nLiteral(node) { + let found = false; + function visit(child) { + if ( + (ts.isStringLiteral(child) || ts.isNoSubstitutionTemplateLiteral(child) || ts.isTemplateHead(child) || ts.isTemplateMiddle(child) || ts.isTemplateTail(child)) && + child.text.includes("i18n:") + ) { + found = true; + } + if (!found) ts.forEachChild(child, visit); + } + if (node) visit(node); + return found; +} + +function isStringLiteralLike(node) { + return ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node); +} + +function isStructuralStringPosition(node, file) { + const parent = node.parent; + if (!parent) return false; + if (ts.isPropertyAssignment(parent) && parent.name === node) return true; + if (ts.isElementAccessExpression(parent) && parent.argumentExpression === node) return true; + if (ts.isLiteralTypeNode(parent)) return true; + if (ts.isImportDeclaration(parent) || ts.isExportDeclaration(parent)) return true; + if (ts.isImportTypeNode(parent)) return true; + if (ts.isCallExpression(parent) && isStructuralCall(parent) && parent.arguments[0] === node) return true; + if (ts.isNewExpression(parent) && parent.expression.getText() === "CustomEvent" && parent.arguments?.[0] === node) return true; + return false; +} + +function isAllowedStructuralStringPosition(node, file) { + if (file.endsWith("/utils/fieldHelp.ts") && ts.isPropertyAssignment(node.parent) && node.parent.name === node) return true; + return hasAncestorVariable(node, "LEGACY_STORAGE_KEYS"); +} + +function isStructuralVariableName(name) { + return /(?:^|_)(?:CLASS|CLASSNAME|STYLE|ID|KEY|SLUG|CODE|PATH|URL|ENDPOINT|ROUTE|SCOPE|PERMISSION|CAPABILITY|EVENT|STORAGE|ALGORITHM)(?:_|$)/i.test(name) || + /(?:ClassName|Style|Id|ID|Key|Slug|Code|Path|Url|URL|Endpoint|Route|Scope|Permission|Capability|Event|Storage|Algorithm)$/.test(name); +} + +function isStructuralCall(node) { + const expression = node.expression.getText(); + return /(?:^|\.)(?:getItem|setItem|removeItem|addEventListener|removeEventListener|dispatchEvent)$/.test(expression); +} + +function isStructuralCallArgument(node, index) { + const expressionName = callExpressionName(node); + if (isStructuralCall(node) && index === 0) return true; + if ((expressionName === "metadataText" || expressionName === "metadataFlag" || expressionName === "getText" || expressionName === "getBool") && index === 1) return true; + if (expressionName === "createAttachmentBasePath" && index === 0) return true; + if (expressionName === "updateNested" && index === 1) return true; + if ((expressionName === "hasScope" || expressionName === "hasAnyScope") && index === 1) return true; + if ((expressionName === "usePlatformModuleInstalled" || expressionName === "usePlatformUiCapability") && index === 0) return true; + return false; +} + +function callExpressionName(node) { + const expression = node.expression; + if (ts.isIdentifier(expression)) return expression.text; + if (ts.isPropertyAccessExpression(expression)) return expression.name.text; + return expression.getText(); +} + +function isLookupArray(node) { + let parent = node.parent; + while (parent && (ts.isParenthesizedExpression(parent) || ts.isAsExpression(parent))) { + parent = parent.parent; + } + return Boolean( + parent && + ts.isPropertyAccessExpression(parent) && + parent.expression === node && + ["includes", "indexOf"].includes(parent.name.text) && + ts.isCallExpression(parent.parent) + ); +} + +function isAlgorithmNameProperty(node) { + if (nodeName(node.name) !== "name") return false; + const objectLiteral = node.parent; + if (!ts.isObjectLiteralExpression(objectLiteral)) return false; + if (objectLiteral.properties.some((property) => ts.isPropertyAssignment(property) && ["hash", "namedCurve"].includes(nodeName(property.name)))) return true; + let parent = objectLiteral.parent; + while (parent && (ts.isConditionalExpression(parent) || ts.isParenthesizedExpression(parent) || ts.isAsExpression(parent))) { + parent = parent.parent; + } + if (ts.isVariableDeclaration(parent)) return /Algorithm$/i.test(nodeName(parent.name)); + if (ts.isPropertyAssignment(parent)) return /Algorithm$/i.test(nodeName(parent.name)); + return false; +} + +function isListOptionValueProperty(node) { + if (nodeName(node.name) !== "value") return false; + const objectLiteral = node.parent; + if (!ts.isObjectLiteralExpression(objectLiteral)) return false; + return objectLiteral.properties.some((property) => ts.isPropertyAssignment(property) && nodeName(property.name) === "label"); +} + +function isStructuralJsxValue(node) { + if (nodeName(node.name) !== "value") return false; + const parent = node.parent; + if (!ts.isJsxOpeningElement(parent) && !ts.isJsxSelfClosingElement(parent)) return false; + return parent.tagName.getText() === "option"; +} + +function hasAncestorVariable(node, variableName) { + let current = node.parent; + while (current) { + if (ts.isVariableDeclaration(current) && nodeName(current.name) === variableName) return true; + current = current.parent; + } + return false; +} + +const structuralNames = new Set([ + "action", + "actionId", + "algorithm", + "allOf", + "anyOf", + "apiKey", + "authMethod", + "capability", + "capabilityId", + "class", + "className", + "code", + "columnType", + "data-testid", + "endpoint", + "event", + "eventName", + "eventType", + "field", + "for", + "form", + "hash", + "href", + "htmlFor", + "iconName", + "id", + "kind", + "key", + "method", + "moduleId", + "nameAttribute", + "namedCurve", + "ownerModule", + "owner_module", + "path", + "permission", + "permissions", + "rel", + "requiredScopes", + "role", + "route", + "scope", + "scopeId", + "scopeType", + "slug", + "status", + "storageKey", + "style", + "target", + "to", + "tone", + "type", + "url", + "variant" +]); + +function loadGeneratedTranslations(file) { + const source = fs.readFileSync(file, "utf8"); + const output = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } + }).outputText; + const sandbox = { exports: {}, module: { exports: {} }, require: () => ({}) }; + vm.runInNewContext(output, sandbox, { filename: file }); + return sandbox.exports.generatedTranslations ?? sandbox.module.exports.generatedTranslations ?? {}; +} + +const cssUnits = /^(?:-?\d+(?:\.\d+)?(?:px|r?em|vh|vw|vmin|vmax|%|s|ms)?|0)(?:\s+(?:-?\d+(?:\.\d+)?(?:px|r?em|vh|vw|vmin|vmax|%|s|ms)?|0)){0,3}$/i; +const cssFunction = /^(?:minmax|repeat|calc|var|clamp|rgba?|hsla?)\(/i; +const color = /^(?:#[0-9a-f]{3,8}|rgba?\([^)]*\)|hsla?\([^)]*\))$/i; +const cssIdentifier = /^-?[_a-zA-Z]+[_a-zA-Z0-9-]*$/; +const emailAddress = /^[^@\s]+@[^@\s]+\.[^@\s]+$/; +const emailAddressList = /^[^@\s;,]+@[^@\s;,]+\.[^@\s;,]+(?:\s*[;,]\s*[^@\s;,]+@[^@\s;,]+\.[^@\s;,]+)*$/; +const urlLike = /^(?:https?|webcal):\/\//i; +const packageSpecifier = /^@[a-z0-9-]+\/[a-z0-9-]+$/i; +const shellCommand = /^(?:systemctl|pg_dump|pg_restore|npm|pnpm|uv|pip|python)\b/; +const wildcardDomain = /^\*\.[a-z0-9.-]+$/i; +const technicalDottedIdentifier = /^[a-z][a-z0-9]*(?:[._:-][a-z0-9][a-z0-9-]*)+$/; +const environmentReference = /^(?:env|secret):[A-Z0-9_]+$/; +const codeEllipsis = /^[a-z][a-z0-9_-]*-\.\.\.$/; +const globOrPathChoice = /^[\w./*-]+(?:\s+or\s+[\w./*-]+)+$/i; +const multiLineStructuralList = /^(?:[A-Za-z0-9_.:/-]+[*?]?(?:\n|$))+$/; +const cryptographicIdentifier = /^(?:ECDSA|Ed25519|RSASSA-PKCS1-v1_5|RSA-PSS|AES-GCM|SHA-\d+)$/; +const classStateTokens = new Set(["active", "subtle", "success", "neutral", "danger", "disabled", "compact", "hidden", "block", "inline"]); +const allowedStructuralLookingValues = new Set(["active tenant", "campaign-local settings", "govoplan-files"]); +const allowedStructuralLookingKeys = /(?:active_tenant|python_package_placeholder|manifest_signature_key|remote_entry|remote_manifest)/; + +function looksStructuralCatalogValue(value, key) { + const trimmed = value.trim(); + if (!trimmed || allowedStructuralLookingValues.has(trimmed) || allowedStructuralLookingKeys.test(key)) return false; + if (emailAddress.test(trimmed) || emailAddressList.test(trimmed) || cryptographicIdentifier.test(trimmed)) return true; + if (environmentReference.test(trimmed) || codeEllipsis.test(trimmed)) return true; + if (globOrPathChoice.test(trimmed) && /[/*.]/.test(trimmed)) return true; + if (urlLike.test(trimmed) || packageSpecifier.test(trimmed) || shellCommand.test(trimmed) || wildcardDomain.test(trimmed)) return true; + if (technicalDottedIdentifier.test(trimmed)) return true; + if (trimmed.includes("\n") && multiLineStructuralList.test(trimmed)) return true; + if (cssUnits.test(trimmed) || cssFunction.test(trimmed) || color.test(trimmed)) return true; + if (/^[.#][A-Za-z_][\w-]*(?:\s*[.#][A-Za-z_][\w-]*)*$/.test(trimmed)) return true; + + const tokens = trimmed.split(/\s+/); + if (!tokens.every((token) => cssIdentifier.test(token))) return false; + if (!tokens.some((token) => token.includes("-") || token.startsWith("is-") || token.startsWith("has-") || classStateTokens.has(token))) return false; + if (value !== trimmed) return true; + if (tokens.length > 1 && tokens.every((token) => token === token.toLowerCase()) && tokens.some((token) => token.includes("-") || token.startsWith("is-") || token.startsWith("has-"))) return true; + if (tokens.length === 1 && (/^(?:is-|has-|with-)/.test(trimmed) || (trimmed.includes("-") && trimmed === trimmed.toLowerCase()))) return true; + return false; +} + +const sourceFindings = scanStructuralSourcePositions(sourceRoots); +const catalogFindings = scanGeneratedCatalogs(generatedCatalogs); +const findings = [...sourceFindings, ...catalogFindings]; + +if (findings.length) { + console.error(findings.join("\n")); + process.exit(1); +} + +console.log("No structural i18n markers found in source positions or generated catalogs."); diff --git a/webui/scripts/test-module-permutations.mjs b/webui/scripts/test-module-permutations.mjs index f37e226..5ba2bc1 100644 --- a/webui/scripts/test-module-permutations.mjs +++ b/webui/scripts/test-module-permutations.mjs @@ -4,8 +4,11 @@ const packageByModule = { access: "@govoplan/access-webui", admin: "@govoplan/admin-webui", campaigns: "@govoplan/campaign-webui", + dashboard: "@govoplan/dashboard-webui", + docs: "@govoplan/docs-webui", files: "@govoplan/files-webui", - mail: "@govoplan/mail-webui" + mail: "@govoplan/mail-webui", + ops: "@govoplan/ops-webui" }; const cases = [ @@ -13,12 +16,14 @@ const cases = [ { name: "access-only", modules: ["access"] }, { name: "admin-only", modules: ["admin"] }, { name: "access-with-admin", modules: ["access", "admin"] }, + { name: "dashboard-only", modules: ["dashboard"] }, { name: "files-only", modules: ["files"] }, { name: "mail-only", modules: ["mail"] }, { name: "campaign-only", modules: ["campaigns"] }, { name: "campaign-with-files-no-mail", modules: ["campaigns", "files"] }, { name: "campaign-with-mail-no-files", modules: ["campaigns", "mail"] }, - { name: "full-product", modules: ["access", "admin", "campaigns", "files", "mail"] } + { name: "docs-and-ops", modules: ["access", "docs", "ops"] }, + { name: "full-product", modules: ["access", "admin", "dashboard", "campaigns", "files", "mail", "docs", "ops"] } ]; const npmExec = process.env.npm_execpath; @@ -27,13 +32,20 @@ const baseArgs = npmExec ? [npmExec, "run", "build"] : ["run", "build"]; for (const testCase of cases) { const packages = testCase.modules.map((moduleId) => packageByModule[moduleId]).join(","); + const env = { + ...process.env, + GOVOPLAN_WEBUI_MODULE_PACKAGES: packages + }; + delete env.npm_config_tmp; + delete env.NPM_CONFIG_TMP; + if (process.env.GOVOPLAN_NPM_USERCONFIG) { + env.NPM_CONFIG_USERCONFIG = process.env.GOVOPLAN_NPM_USERCONFIG; + delete env.npm_config_userconfig; + } console.log(`\n== WebUI module permutation: ${testCase.name} ==`); const result = spawnSync(command, baseArgs, { cwd: new URL("..", import.meta.url), - env: { - ...process.env, - GOVOPLAN_WEBUI_MODULE_PACKAGES: packages - }, + env, stdio: "inherit" }); if (result.status !== 0) { diff --git a/webui/src/App.tsx b/webui/src/App.tsx index 480447f..c94a2f2 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -1,34 +1,46 @@ import { Navigate, Route, Routes } from "react-router-dom"; import { lazy, Suspense, useEffect, useMemo, useState } from "react"; -import { fetchMe } from "./api/auth"; +import { fetchMe, updateProfile } from "./api/auth"; import { fetchPlatformModules, fetchPlatformStatus } from "./api/platform"; import { AUTH_REQUIRED_EVENT, loadApiSettings, saveApiSettings, type AuthRequiredEventDetail } from "./api/client"; -import type { ApiSettings, AuthInfo, LoginResponse, PlatformModuleInfo, PlatformWebModule } from "./types"; +import type { ApiSettings, AuthInfo, AuthTenant, AuthTenantMembership, AuthUser, LoginResponse, PlatformModuleInfo, PlatformWebModule, UserUiPreferences } from "./types"; import AppShell from "./layout/AppShell"; import PublicLandingPage from "./features/auth/PublicLandingPage"; import LoginModal from "./features/auth/LoginModal"; import { PermissionBoundary } from "./components/AccessBoundary"; -import { firstAccessibleRoute, loadRemoteWebModules, navItemsForModules, resolveInstalledWebModules, routeContributionsForModules } from "./platform/modules"; +import { firstAccessibleRoute, loadRemoteWebModules, moduleInstalled, navItemsForModules, resolveInstalledWebModules, routeContributionsForModules } from "./platform/modules"; import { PlatformModulesProvider } from "./platform/ModuleContext"; import { PLATFORM_MODULES_CHANGED_EVENT } from "./platform/moduleEvents"; import { UnsavedChangesProvider } from "./components/UnsavedChangesGuard"; +import { PlatformLanguageProvider, type PlatformLanguage } from "./i18n/LanguageContext"; const DashboardPage = lazy(() => import("./features/dashboard/DashboardPage")); const SettingsPage = lazy(() => import("./features/settings/SettingsPage")); +const DEFAULT_UI_PREFERENCES: UserUiPreferences = { + compact_tables: false, + show_inline_help_hints: true, + reduce_motion: false, + sticky_section_sidebars: true, + theme: "system" +}; + export default function App() { const [settings, setSettings] = useState(() => loadApiSettings()); const [auth, setAuth] = useState(null); const [checkingSession, setCheckingSession] = useState(true); const [platformModules, setPlatformModules] = useState(null); const [remoteWebModules, setRemoteWebModules] = useState([]); - const [maintenanceMode, setMaintenanceMode] = useState<{ enabled: boolean; message?: string | null }>({ enabled: false, message: null }); + const [maintenanceMode, setMaintenanceMode] = useState<{enabled: boolean;message?: string | null;}>({ enabled: false, message: null }); + const [systemLanguages, setSystemLanguages] = useState<{available: PlatformLanguage[];enabled: string[];defaultLanguage: string;} | null>(null); const [reloginMessage, setReloginMessage] = useState(""); const localWebModules = useMemo(() => resolveInstalledWebModules(platformModules), [platformModules]); const webModules = useMemo(() => mergeWebModules(localWebModules, remoteWebModules), [localWebModules, remoteWebModules]); const navItems = useMemo(() => navItemsForModules(webModules), [webModules]); const moduleRoutes = useMemo(() => routeContributionsForModules(webModules), [webModules]); + const moduleTranslations = useMemo(() => webModules.map((module) => module.translations).filter(Boolean), [webModules]); + const dashboardModuleInstalled = useMemo(() => moduleInstalled("dashboard", webModules), [webModules]); function updateSettings(next: ApiSettings) { setSettings(next); @@ -37,7 +49,7 @@ export default function App() { function updateAuth(next: AuthInfo | null, accessToken?: string) { const nextSettings = accessToken !== undefined ? { ...settings, accessToken } : settings; - setAuth(next); + setAuth(next ? normalizeAuthInfo(next) : null); if (accessToken !== undefined) { setSettings(nextSettings); saveApiSettings(nextSettings); @@ -45,16 +57,7 @@ export default function App() { } function authFromLoginResponse(response: LoginResponse): AuthInfo { - const active = response.active_tenant ?? response.tenant; - return { - user: response.user, - tenant: active, - active_tenant: active, - tenants: response.tenants ?? [active], - scopes: response.scopes, - roles: response.roles, - groups: response.groups - }; + return normalizeAuthInfo(response); } function handlePublicLogin(response: LoginResponse) { @@ -70,7 +73,7 @@ export default function App() { function handleAuthRequired(event: Event) { if (!auth) return; const detail = (event as CustomEvent).detail; - setReloginMessage(detail?.message || "Your session has expired. Sign in again to continue."); + setReloginMessage(detail?.message || "i18n:govoplan-core.your_session_has_expired_sign_in_again_to_contin.4cc6a1b3"); } window.addEventListener(AUTH_REQUIRED_EVENT, handleAuthRequired); @@ -79,35 +82,57 @@ export default function App() { useEffect(() => { let cancelled = false; - fetchPlatformStatus(settings) - .then((response) => { - if (!cancelled) setMaintenanceMode(response.maintenance_mode); - }) - .catch(() => { + + async function refreshPlatformStatus() { + try { + const response = await fetchPlatformStatus(settings); + if (cancelled) return; + setMaintenanceMode(response.maintenance_mode); + if (response.i18n) { + setSystemLanguages({ + available: response.i18n.available_languages.map((item) => ({ + code: item.code, + label: item.label, + nativeLabel: item.native_label ?? undefined + })), + enabled: response.i18n.enabled_languages, + defaultLanguage: response.i18n.default_language + }); + } + } catch { if (!cancelled) setMaintenanceMode({ enabled: false, message: null }); - }); - return () => { cancelled = true; }; + } + } + + void refreshPlatformStatus(); + const interval = window.setInterval(() => {void refreshPlatformStatus();}, 30_000); + window.addEventListener("focus", refreshPlatformStatus); + return () => { + cancelled = true; + window.clearInterval(interval); + window.removeEventListener("focus", refreshPlatformStatus); + }; }, [settings.apiBaseUrl]); useEffect(() => { let cancelled = false; setCheckingSession(true); - fetchMe(settings) - .then((me) => { if (!cancelled) setAuth(me); }) - .catch(() => { - if (!cancelled) { - const cleared = { ...settings, accessToken: "" }; - setSettings(cleared); - saveApiSettings(cleared); - setAuth(null); - setPlatformModules(null); - setRemoteWebModules([]); - } - }) - .finally(() => { - if (!cancelled) setCheckingSession(false); - }); - return () => { cancelled = true; }; + fetchMe(settings). + then((me) => {if (!cancelled) setAuth(normalizeAuthInfo(me));}). + catch(() => { + if (!cancelled) { + const cleared = { ...settings, accessToken: "" }; + setSettings(cleared); + saveApiSettings(cleared); + setAuth(null); + setPlatformModules(null); + setRemoteWebModules([]); + } + }). + finally(() => { + if (!cancelled) setCheckingSession(false); + }); + return () => {cancelled = true;}; }, [settings.apiBaseUrl, settings.apiKey]); useEffect(() => { @@ -119,10 +144,10 @@ export default function App() { function loadModules() { inFlight = true; lastRefreshAt = Date.now(); - return fetchPlatformModules(settings) - .then((response) => { if (!cancelled) setPlatformModules(response.modules); }) - .catch(() => { if (!cancelled) setPlatformModules(null); }) - .finally(() => { inFlight = false; }); + return fetchPlatformModules(settings). + then((response) => {if (!cancelled) setPlatformModules(response.modules);}). + catch(() => {if (!cancelled) setPlatformModules(null);}). + finally(() => {inFlight = false;}); } void loadModules(); @@ -146,19 +171,39 @@ export default function App() { window.removeEventListener("focus", refreshVisibleModules); document.removeEventListener("visibilitychange", refreshVisibleModules); }; - }, [auth?.user.id, auth?.active_tenant?.id, auth?.tenant.id, settings.apiBaseUrl, settings.apiKey]); + }, [auth?.user?.id, auth?.active_tenant?.id, auth?.tenant.id, settings.apiBaseUrl, settings.apiKey]); + + useEffect(() => { + const preferences = auth?.user.ui_preferences ?? DEFAULT_UI_PREFERENCES; + const root = document.documentElement; + root.classList.toggle("ui-compact-tables", preferences.compact_tables); + root.classList.toggle("ui-hide-help-hints", !preferences.show_inline_help_hints); + root.classList.toggle("ui-reduce-motion", preferences.reduce_motion); + root.classList.toggle("ui-no-sticky-section-sidebars", !preferences.sticky_section_sidebars); + if (preferences.theme === "system") { + delete root.dataset.theme; + } else { + root.dataset.theme = preferences.theme; + } + }, [ + auth?.user.ui_preferences?.compact_tables, + auth?.user.ui_preferences?.show_inline_help_hints, + auth?.user.ui_preferences?.reduce_motion, + auth?.user.ui_preferences?.sticky_section_sidebars, + auth?.user.ui_preferences?.theme + ]); useEffect(() => { let cancelled = false; if (!auth || !platformModules?.length) { setRemoteWebModules([]); - return () => { cancelled = true; }; + return () => {cancelled = true;}; } - loadRemoteWebModules(platformModules, localWebModules) - .then((modules) => { if (!cancelled) setRemoteWebModules(modules); }) - .catch(() => { if (!cancelled) setRemoteWebModules([]); }); - return () => { cancelled = true; }; - }, [auth?.user.id, auth?.active_tenant?.id, auth?.tenant.id, platformModules, localWebModules]); + loadRemoteWebModules(platformModules, localWebModules). + then((modules) => {if (!cancelled) setRemoteWebModules(modules);}). + catch(() => {if (!cancelled) setRemoteWebModules([]);}); + return () => {cancelled = true;}; + }, [auth?.user?.id, auth?.active_tenant?.id, auth?.tenant.id, platformModules, localWebModules]); useEffect(() => { if (!auth) return; @@ -174,12 +219,12 @@ export default function App() { inFlight = true; lastRefreshAt = now; try { - setAuth(await fetchMe(settings)); + setAuth(normalizeAuthInfo(await fetchMe(settings))); } catch { + + // A background refresh must not log the user out on a transient network error. - } finally { - inFlight = false; - } + } finally {inFlight = false;} } window.addEventListener("focus", refreshVisibleSession); @@ -188,87 +233,189 @@ export default function App() { window.removeEventListener("focus", refreshVisibleSession); document.removeEventListener("visibilitychange", refreshVisibleSession); }; - }, [auth?.user.id, auth?.active_tenant?.id, auth?.tenant.id, settings.apiBaseUrl, settings.apiKey]); + }, [auth?.user?.id, auth?.active_tenant?.id, auth?.tenant.id, settings.apiBaseUrl, settings.apiKey]); if (checkingSession) { return ( + - +
-
GovOPlaN
-

Checking session...

-

Please wait while the local session is verified.

+
i18n:govoplan-core.govoplan.a84c0a85
+

i18n:govoplan-core.checking_session.e7d81968

+

i18n:govoplan-core.please_wait_while_the_local_session_is_verified.1dbd8fd3

 - ); + ); + } if (!auth) { return ( + - + - ); + ); + } const defaultRoute = firstAccessibleRoute(auth, webModules); + const authAvailableLanguages = auth.available_languages?.map((item) => ({ + code: item.code, + label: item.label, + nativeLabel: item.native_label ?? undefined + })); + + function persistLanguagePreference(code: string) { + void updateProfile(settings, { preferred_language: code }).then((next) => updateAuth(next)).catch(() => undefined); + } return ( + - -

Loading module...

}> + +

i18n:govoplan-core.loading_module.50161f3c

}> } /> - } /> - {moduleRoutes.map((route) => ( - + {!dashboardModuleInstalled && } />} + {moduleRoutes.map((route) => + {route.render({ settings, auth, onAuthChange: updateAuth })} - } - /> - ))} + } /> + + )} } /> } /> - {reloginMessage && ( - setReloginMessage("")} - onLogin={handleRelogin} - /> - )} + {reloginMessage && + setReloginMessage("")} + onLogin={handleRelogin} /> + + } - ); + ); + +} + +type AuthPayload = Partial & { + principal?: AuthInfo["principal"]; + user?: Partial | null; + tenant?: AuthTenant | null; + active_tenant?: AuthTenant | null; + tenants?: AuthTenantMembership[] | null; +}; + +function normalizeAuthInfo(response: AuthPayload): AuthInfo { + const principal = response.principal ?? null; + const activeTenant = response.active_tenant ?? response.tenant ?? response.tenants?.[0] ?? null; + const user = normalizeAuthUser(response.user, principal); + + if (!activeTenant) { + throw new Error("Authentication response did not include an active tenant."); + } + if (!user) { + throw new Error("Authentication response did not include a user."); + } + + return { + user, + tenant: activeTenant, + active_tenant: activeTenant, + tenants: response.tenants ?? [activeTenant], + scopes: response.scopes ?? principal?.scopes ?? [], + roles: response.roles ?? [], + groups: response.groups ?? [], + principal, + available_languages: response.available_languages ?? [], + enabled_language_codes: response.enabled_language_codes ?? activeTenant.enabled_language_codes ?? [], + default_language: response.default_language ?? user.preferred_language ?? activeTenant.default_locale ?? "en" + }; +} + +function normalizeAuthUser(user: Partial | null | undefined, principal: AuthInfo["principal"]): AuthUser | null { + if (user?.id && user.account_id) { + return { + id: user.id, + account_id: user.account_id, + email: user.email ?? principal?.email ?? "", + display_name: user.display_name ?? principal?.display_name ?? principal?.email ?? null, + tenant_display_name: user.tenant_display_name ?? null, + is_tenant_admin: user.is_tenant_admin ?? false, + password_reset_required: user.password_reset_required ?? false, + preferred_language: user.preferred_language ?? null, + enabled_language_codes: user.enabled_language_codes ?? [], + ui_preferences: normalizeUiPreferences(user.ui_preferences) + }; + } + + if (!principal?.membership_id || !principal.account_id) { + return null; + } + + return { + id: principal.membership_id, + account_id: principal.account_id, + email: principal.email ?? "", + display_name: principal.display_name ?? principal.email ?? null, + tenant_display_name: principal.display_name ?? null, + is_tenant_admin: false, + password_reset_required: false, + preferred_language: null, + enabled_language_codes: [], + ui_preferences: DEFAULT_UI_PREFERENCES + }; +} + +function normalizeUiPreferences(value: Partial | null | undefined): UserUiPreferences { + const theme = value?.theme === "light" || value?.theme === "dark" || value?.theme === "system" ? value.theme : "system"; + return { + compact_tables: Boolean(value?.compact_tables ?? DEFAULT_UI_PREFERENCES.compact_tables), + show_inline_help_hints: Boolean(value?.show_inline_help_hints ?? DEFAULT_UI_PREFERENCES.show_inline_help_hints), + reduce_motion: Boolean(value?.reduce_motion ?? DEFAULT_UI_PREFERENCES.reduce_motion), + sticky_section_sidebars: Boolean(value?.sticky_section_sidebars ?? DEFAULT_UI_PREFERENCES.sticky_section_sidebars), + theme + }; } function mergeWebModules(localModules: PlatformWebModule[], remoteModules: PlatformWebModule[]): PlatformWebModule[] { if (remoteModules.length === 0) return localModules; const seen = new Set(localModules.map((module) => module.id)); return [ - ...localModules, - ...remoteModules.filter((module) => { - if (seen.has(module.id)) return false; - seen.add(module.id); - return true; - }) - ]; + ...localModules, + ...remoteModules.filter((module) => { + if (seen.has(module.id)) return false; + seen.add(module.id); + return true; + })]; + } diff --git a/webui/src/api/admin.ts b/webui/src/api/admin.ts index 739ab5c..1fe145e 100644 --- a/webui/src/api/admin.ts +++ b/webui/src/api/admin.ts @@ -150,11 +150,20 @@ export type PrivacyRetentionPolicyScope = "system" | "tenant" | "user" | "group" export type PolicySourceStep = { scope_type: string; scope_id?: string | null; + path: string; label: string; applied_fields?: string[]; policy?: PrivacyRetentionPolicyPatch | PrivacyRetentionPolicy | null; }; +export type PolicyDecision = { + allowed: boolean; + reason?: string | null; + source_path: PolicySourceStep[]; + requirements: string[]; + details?: Record; +}; + export type PrivacyRetentionPolicyScopeResponse = { scope_type: PrivacyRetentionPolicyScope; scope_id?: string | null; @@ -165,20 +174,43 @@ export type PrivacyRetentionPolicyScopeResponse = { parent_policy_sources?: PolicySourceStep[]; }; +export type PrivacyRetentionPolicyExplainResponse = { + scope_type: PrivacyRetentionPolicyScope; + scope_id?: string | null; + decision: PolicyDecision; + effective_policy: PrivacyRetentionPolicy; + parent_policy?: PrivacyRetentionPolicy | null; + effective_policy_sources?: PolicySourceStep[]; + parent_policy_sources?: PolicySourceStep[]; + blocked_fields: PrivacyRetentionPolicyFieldKey[]; +}; + export type SystemSettingsItem = { default_locale: string; allow_tenant_custom_groups: boolean; allow_tenant_custom_roles: boolean; allow_tenant_api_keys: boolean; privacy_retention_policy: PrivacyRetentionPolicy; + maintenance_mode?: { enabled: boolean; message?: string | null }; + available_languages?: LanguagePackage[]; + enabled_language_codes?: string[]; settings: Record; }; +export type LanguagePackage = { + code: string; + label: string; + native_label?: string | null; +}; + export type TenantSettingsItem = { id: string; slug: string; name: string; default_locale: string; + available_languages: LanguagePackage[]; + system_enabled_language_codes: string[]; + enabled_language_codes: string[]; settings: Record; }; @@ -192,6 +224,116 @@ export type RetentionRunResponse = { }; }; +export type ConfigurationSafetyField = { + key: string; + label: string; + owner_module: string; + scope: "system" | "tenant" | "user" | "group" | "campaign"; + storage: string; + ui_managed: boolean; + risk: "low" | "medium" | "high" | "destructive"; + secret_handling: "none" | "reference_only" | "env_only"; + required_scopes: string[]; + dry_run_required: boolean; + validation_required: boolean; + policy_explanation_required: boolean; + audit_event?: string | null; + maintenance_required: boolean; + two_person_approval_required: boolean; + rollback_history_required: boolean; + notes?: string | null; +}; + +export type ConfigurationChangeSafetyPlan = { + key: string; + allowed: boolean; + field?: ConfigurationSafetyField | null; + risk?: ConfigurationSafetyField["risk"] | null; + missing_scopes: string[]; + dry_run_required: boolean; + dry_run_satisfied: boolean; + approval_required: boolean; + approval_satisfied: boolean; + maintenance_required: boolean; + maintenance_satisfied: boolean; + rollback_history_required: boolean; + secret_handling: ConfigurationSafetyField["secret_handling"]; + audit_event?: string | null; + policy_explanation?: string | null; + blockers: string[]; + warnings: string[]; +}; + +export type ConfigurationChangeRequest = { + id: string; + key: string; + label?: string; + target?: Record; + dry_run: boolean; + requested_by: string; + requested_at: string; + updated_at: string; + status: "pending_approval" | "approved" | "applied" | "rejected" | string; + approvals: Array>; + plan: ConfigurationChangeSafetyPlan; + value_preview?: unknown; +}; + +export type ConfigurationChangeRecord = { + id: string; + version: number; + key: string; + target?: Record; + actor_user_id: string; + approval_request_id?: string | null; + approval_user_ids: string[]; + before?: unknown; + after?: unknown; + rollback_value?: unknown; + status: string; + created_at: string; +}; + +export type ConfigurationPackageDiagnostic = { + severity: "blocker" | "warning" | "info"; + code: string; + message: string; + module_id?: string | null; + object_ref?: string | null; + resolution?: string | null; +}; + +export type ConfigurationPackageRequiredData = { + key: string; + label: string; + data_type: string; + required: boolean; + secret: boolean; + description?: string | null; +}; + +export type ConfigurationPackagePlanItem = { + action: "create" | "update" | "bind" | "skip" | "blocked" | "noop"; + module_id: string; + fragment_type: string; + fragment_id?: string | null; + summary?: string | null; +}; + +export type ConfigurationPackageFragment = { + module_id: string; + fragment_type: string; + fragment_id?: string | null; + payload: Record; +}; + +export type ConfigurationPackageRunPayload = { + package: Record; + tenant_id?: string | null; + supplied_data?: Record; + change_request_id?: string | null; +}; + export type GovernanceAssignment = { tenant_id: string; mode: "available" | "required"; @@ -289,7 +431,7 @@ export function fetchTenantSettings(settings: ApiSettings): Promise { +export function updateTenantSettings(settings: ApiSettings, payload: { default_locale: string; enabled_language_codes?: string[] | null }): Promise { return apiFetch(settings, "/api/v1/admin/tenant/settings", { method: "PATCH", body: JSON.stringify(payload) }); } @@ -502,6 +644,10 @@ export type SystemSettingsUpdatePayload = { allow_tenant_custom_roles: boolean; allow_tenant_api_keys: boolean; privacy_retention_policy?: PrivacyRetentionPolicy | null; + maintenance_mode?: { enabled: boolean; message?: string | null } | null; + available_languages?: LanguagePackage[] | null; + enabled_language_codes?: string[] | null; + change_request_id?: string | null; }; export function updateSystemSettings(settings: ApiSettings, payload: SystemSettingsUpdatePayload): Promise { @@ -515,31 +661,127 @@ export function getPrivacyRetentionPolicy(settings: ApiSettings, scope: PrivacyR return apiFetch(settings, `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}${suffix}`); } -export function updatePrivacyRetentionPolicy(settings: ApiSettings, scope: PrivacyRetentionPolicyScope, policy: PrivacyRetentionPolicyPatch, scopeId?: string | null): Promise { +export function explainPrivacyRetentionPolicy(settings: ApiSettings, scope: PrivacyRetentionPolicyScope, scopeId?: string | null): Promise { const params = new URLSearchParams(); if (scopeId) params.set("scope_id", scopeId); const suffix = params.toString() ? `?${params.toString()}` : ""; - return apiFetch(settings, `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}${suffix}`, { method: "PUT", body: JSON.stringify({ policy }) }); + return apiFetch(settings, `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}/explain${suffix}`); +} + +export function updatePrivacyRetentionPolicy(settings: ApiSettings, scope: PrivacyRetentionPolicyScope, policy: PrivacyRetentionPolicyPatch, scopeId?: string | null, changeRequestId?: string | null): Promise { + const params = new URLSearchParams(); + if (scopeId) params.set("scope_id", scopeId); + const suffix = params.toString() ? `?${params.toString()}` : ""; + return apiFetch(settings, `/api/v1/admin/privacy-retention/policies/${encodeURIComponent(scope)}${suffix}`, { method: "PUT", body: JSON.stringify({ policy, change_request_id: changeRequestId ?? null }) }); } export function runRetentionPolicy(settings: ApiSettings, dryRun = true): Promise { return apiFetch(settings, "/api/v1/admin/system/retention/run", { method: "POST", body: JSON.stringify({ dry_run: dryRun }) }); } +export async function fetchConfigurationSafetyCatalog(settings: ApiSettings, includeEnvOnly = false): Promise { + const suffix = includeEnvOnly ? "?include_env_only=true" : ""; + const response = await apiFetch<{ fields: ConfigurationSafetyField[] }>(settings, `/api/v1/admin/configuration-safety${suffix}`); + return response.fields; +} + +export async function planConfigurationChange(settings: ApiSettings, payload: { + key: string; + value?: unknown; + dry_run?: boolean; + maintenance_mode?: boolean; + approval_count?: number; +}): Promise { + const response = await apiFetch<{ plan: ConfigurationChangeSafetyPlan }>(settings, "/api/v1/admin/configuration-safety/plan", { + method: "POST", + body: JSON.stringify(payload) + }); + return response.plan; +} + +export function fetchConfigurationChanges(settings: ApiSettings): Promise<{ requests: ConfigurationChangeRequest[]; history: ConfigurationChangeRecord[] }> { + return apiFetch(settings, "/api/v1/admin/configuration-changes"); +} + +export async function createConfigurationChangeRequest(settings: ApiSettings, payload: { + key: string; + value?: unknown; + dry_run?: boolean; + target?: Record; + reason?: string | null; +}): Promise { + const response = await apiFetch<{ request: ConfigurationChangeRequest }>(settings, "/api/v1/admin/configuration-change-requests", { + method: "POST", + body: JSON.stringify(payload) + }); + return response.request; +} + +export async function approveConfigurationChangeRequest(settings: ApiSettings, requestId: string, reason?: string | null): Promise { + const response = await apiFetch<{ request: ConfigurationChangeRequest }>(settings, `/api/v1/admin/configuration-change-requests/${encodeURIComponent(requestId)}/approve`, { + method: "POST", + body: JSON.stringify({ reason: reason ?? null }) + }); + return response.request; +} + +export function fetchConfigurationPackageCatalogValidation(settings: ApiSettings): Promise<{ validation: Record }> { + return apiFetch(settings, "/api/v1/admin/configuration-packages/catalog"); +} + +export function dryRunConfigurationPackage(settings: ApiSettings, payload: ConfigurationPackageRunPayload): Promise<{ + diagnostics: ConfigurationPackageDiagnostic[]; + required_data: ConfigurationPackageRequiredData[]; + plan: ConfigurationPackagePlanItem[]; +}> { + return apiFetch(settings, "/api/v1/admin/configuration-packages/dry-run", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function applyConfigurationPackage(settings: ApiSettings, payload: ConfigurationPackageRunPayload): Promise<{ + diagnostics: ConfigurationPackageDiagnostic[]; + created_refs: Record; + updated_refs: Record; +}> { + return apiFetch(settings, "/api/v1/admin/configuration-packages/apply", { + method: "POST", + body: JSON.stringify(payload) + }); +} + +export function exportConfigurationPackage(settings: ApiSettings, payload: { + tenant_id?: string | null; + scopes?: string[]; + module_ids?: string[]; + object_refs?: string[]; +}): Promise<{ + fragments: ConfigurationPackageFragment[]; + data_requirements: ConfigurationPackageRequiredData[]; + diagnostics: ConfigurationPackageDiagnostic[]; +}> { + return apiFetch(settings, "/api/v1/admin/configuration-packages/export", { + method: "POST", + body: JSON.stringify(payload) + }); +} + export async function fetchGovernanceTemplates(settings: ApiSettings, kind?: "group" | "role"): Promise { const suffix = kind ? `?kind=${encodeURIComponent(kind)}` : ""; const response = await apiFetch<{ templates: GovernanceTemplateItem[] }>(settings, `/api/v1/admin/system/governance-templates${suffix}`); return response.templates; } -export function createGovernanceTemplate(settings: ApiSettings, payload: Omit): Promise { +export function createGovernanceTemplate(settings: ApiSettings, payload: Omit & { change_request_id?: string | null }): Promise { return apiFetch(settings, "/api/v1/admin/system/governance-templates", { method: "POST", body: JSON.stringify(payload) }); } -export function updateGovernanceTemplate(settings: ApiSettings, templateId: string, payload: Omit): Promise { +export function updateGovernanceTemplate(settings: ApiSettings, templateId: string, payload: Omit & { change_request_id?: string | null }): Promise { return apiFetch(settings, `/api/v1/admin/system/governance-templates/${templateId}`, { method: "PATCH", body: JSON.stringify(payload) }); } -export function deleteGovernanceTemplate(settings: ApiSettings, templateId: string): Promise { - return apiFetch(settings, `/api/v1/admin/system/governance-templates/${templateId}`, { method: "DELETE" }); +export function deleteGovernanceTemplate(settings: ApiSettings, templateId: string, changeRequestId?: string | null): Promise { + const suffix = changeRequestId ? `?change_request_id=${encodeURIComponent(changeRequestId)}` : ""; + return apiFetch(settings, `/api/v1/admin/system/governance-templates/${templateId}${suffix}`, { method: "DELETE" }); } diff --git a/webui/src/api/auth.ts b/webui/src/api/auth.ts index 9a441c5..685e6c2 100644 --- a/webui/src/api/auth.ts +++ b/webui/src/api/auth.ts @@ -1,4 +1,4 @@ -import type { ApiSettings, AuthInfo, LoginResponse } from "../types"; +import type { ApiSettings, AuthInfo, LoginResponse, UserUiPreferences } from "../types"; import { apiFetch } from "./client"; export async function login( @@ -28,7 +28,13 @@ export async function logout(settings: ApiSettings): Promise { export async function updateProfile( settings: ApiSettings, - payload: { display_name?: string | null; tenant_display_name?: string | null } + payload: { + display_name?: string | null; + tenant_display_name?: string | null; + preferred_language?: string | null; + enabled_language_codes?: string[] | null; + ui_preferences?: Partial | null; + } ): Promise { return apiFetch(settings, "/api/v1/auth/profile", { method: "PATCH", diff --git a/webui/src/api/client.ts b/webui/src/api/client.ts index 2909ea5..a22aa52 100644 --- a/webui/src/api/client.ts +++ b/webui/src/api/client.ts @@ -1,10 +1,12 @@ import type { ApiSettings } from "../types"; const STORAGE_KEY = "multimailer.apiSettings"; +const LEGACY_STORAGE_KEYS = ["i18n:govoplan-core.multimailer_apisettings.1d1601d4"]; const SESSION_STORAGE_KEY = "multimailer.session"; const CSRF_COOKIE_NAME = import.meta.env.VITE_CSRF_COOKIE_NAME ?? "msm_csrf"; const RECENT_SAFE_REQUEST_TTL_MS = 750; const MAX_RECENT_SAFE_REQUESTS = 100; +const MAX_CONDITIONAL_SAFE_REQUESTS = 200; export const AUTH_REQUIRED_EVENT = "govoplan:auth-required"; @@ -19,8 +21,14 @@ type RecentSafeRequest = { expiresAt: number; }; +type ConditionalSafeRequest = { + value: unknown; + etag: string; +}; + const inFlightSafeRequests = new Map>(); const recentSafeRequests = new Map(); +const conditionalSafeRequests = new Map(); let safeRequestGeneration = 0; export class ApiError extends Error { @@ -47,8 +55,20 @@ export function isApiError(error: unknown, ...statuses: number[]): error is ApiE * API prefix so old localStorage settings cannot produce /api/v1/api/v1 URLs. */ export function normalizeApiBaseUrl(value: string): string { - const withoutTrailingSlash = value.trim().replace(/\/+$/, ""); - return withoutTrailingSlash.replace(/\/api(?:\/v1)?$/i, ""); + const trimmed = value.trim(); + if (!trimmed || trimmed === "." || trimmed === "./") return ""; + + const withoutTrailingSlash = trimmed.replace(/\/+$/, ""); + const withoutApiPrefix = withoutTrailingSlash.replace(/\/api(?:\/v1)?$/i, ""); + + // API base is an origin, not a frontend route prefix. Path-only values such + // as /a or . are stale/invalid local settings and would bypass Vite's /api + // proxy by producing /a/api/v1/... requests. + if (withoutApiPrefix.startsWith("/") && !withoutApiPrefix.startsWith("//")) { + return ""; + } + + return withoutApiPrefix; } export function apiUrl(settings: ApiSettings, path: string): string { @@ -58,7 +78,8 @@ export function apiUrl(settings: ApiSettings, path: string): string { } export function loadApiSettings(): ApiSettings { - const storedBaseUrl = localStorage.getItem(`${STORAGE_KEY}.baseUrl`); + const storedBaseUrl = loadStoredSetting("baseUrl"); + const storedApiKey = loadStoredSetting("apiKey"); const configuredBaseUrl = storedBaseUrl !== null ? storedBaseUrl : import.meta.env.VITE_API_BASE_URL ?? ""; const apiBaseUrl = normalizeApiBaseUrl(configuredBaseUrl); @@ -66,6 +87,11 @@ export function loadApiSettings(): ApiSettings { if (storedBaseUrl !== null && storedBaseUrl !== apiBaseUrl) { localStorage.setItem(`${STORAGE_KEY}.baseUrl`, apiBaseUrl); } + if (storedApiKey !== null) { + localStorage.setItem(`${STORAGE_KEY}.apiKey`, storedApiKey); + } + clearLegacyStoredSetting("baseUrl"); + clearLegacyStoredSetting("apiKey"); localStorage.removeItem(`${STORAGE_KEY}.accessToken`); sessionStorage.removeItem(`${SESSION_STORAGE_KEY}.accessToken`); @@ -73,7 +99,7 @@ export function loadApiSettings(): ApiSettings { return { // Empty base URL means "same origin". In Vite dev, /api is proxied to FastAPI. apiBaseUrl, - apiKey: localStorage.getItem(`${STORAGE_KEY}.apiKey`) || "", + apiKey: storedApiKey || "", accessToken: "" }; } @@ -81,6 +107,8 @@ export function loadApiSettings(): ApiSettings { export function saveApiSettings(settings: ApiSettings): void { localStorage.setItem(`${STORAGE_KEY}.baseUrl`, normalizeApiBaseUrl(settings.apiBaseUrl)); localStorage.setItem(`${STORAGE_KEY}.apiKey`, settings.apiKey); + clearLegacyStoredSetting("baseUrl"); + clearLegacyStoredSetting("apiKey"); localStorage.removeItem(`${STORAGE_KEY}.accessToken`); sessionStorage.removeItem(`${SESSION_STORAGE_KEY}.accessToken`); } @@ -90,14 +118,29 @@ export function clearAccessToken(): void { localStorage.removeItem(`${STORAGE_KEY}.accessToken`); } +function loadStoredSetting(name: string): string | null { + const keys = [STORAGE_KEY, ...LEGACY_STORAGE_KEYS]; + for (const key of keys) { + const value = localStorage.getItem(`${key}.${name}`); + if (value !== null) return value; + } + return null; +} + +function clearLegacyStoredSetting(name: string): void { + for (const key of LEGACY_STORAGE_KEYS) { + localStorage.removeItem(`${key}.${name}`); + } +} + function readCookie(name: string): string { const prefix = `${encodeURIComponent(name)}=`; - return document.cookie - .split(";") - .map((part) => part.trim()) - .find((part) => part.startsWith(prefix)) - ?.slice(prefix.length) ?? ""; + return document.cookie. + split(";"). + map((part) => part.trim()). + find((part) => part.startsWith(prefix))?. + slice(prefix.length) ?? ""; } export function csrfToken(): string { @@ -111,29 +154,29 @@ function isUnsafeMethod(method?: string): boolean { } function canReuseSafeRequest(method: string, init?: RequestInit): boolean { - return (method === "GET" || method === "HEAD") - && !init?.body - && !init?.signal - && init?.cache !== "no-store" - && init?.cache !== "reload"; + return (method === "GET" || method === "HEAD") && + !init?.body && + !init?.signal && + init?.cache !== "no-store" && + init?.cache !== "reload"; } function requestHeadersKey(headers: Headers): string { - return [...headers.entries()] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, value]) => `${key}:${value}`) - .join("\n"); + return [...headers.entries()]. + sort(([left], [right]) => left.localeCompare(right)). + map(([key, value]) => `${key}:${value}`). + join("\n"); } function safeRequestKey(url: string, method: string, headers: Headers, init?: RequestInit): string { return [ - method, - url, - init?.credentials ?? "include", - init?.mode ?? "", - init?.redirect ?? "", - requestHeadersKey(headers), - ].join("\n\n"); + method, + url, + init?.credentials ?? "include", + init?.mode ?? "", + init?.redirect ?? "", + requestHeadersKey(headers)]. + join("\n\n"); } function pruneRecentSafeRequests(now = Date.now()): void { @@ -167,6 +210,31 @@ function rememberSafeResponse(key: string, value: unknown): void { pruneRecentSafeRequests(now); } +function conditionalSafeResponse(key: string): ConditionalSafeRequest | undefined { + const cached = conditionalSafeRequests.get(key); + if (!cached) return undefined; + conditionalSafeRequests.delete(key); + conditionalSafeRequests.set(key, cached); + return cached; +} + +function rememberConditionalSafeResponse(key: string, etag: string, value: unknown): void { + conditionalSafeRequests.delete(key); + conditionalSafeRequests.set(key, { etag, value }); + while (conditionalSafeRequests.size > MAX_CONDITIONAL_SAFE_REQUESTS) { + const oldestKey = conditionalSafeRequests.keys().next().value; + if (!oldestKey) break; + conditionalSafeRequests.delete(oldestKey); + } +} + +function clearSafeRequestCaches(): void { + safeRequestGeneration += 1; + inFlightSafeRequests.clear(); + recentSafeRequests.clear(); + conditionalSafeRequests.clear(); +} + export function authHeaders(settings: ApiSettings): Headers { const headers = new Headers(); if (settings.accessToken) { @@ -187,13 +255,13 @@ function notifyAuthRequired(path: string): void { const detail: AuthRequiredEventDetail = { path, status: 401, - message: "Your session has expired. Sign in again to continue." + message: "i18n:govoplan-core.your_session_has_expired_sign_in_again_to_contin.4cc6a1b3" }; window.dispatchEvent(new CustomEvent(AUTH_REQUIRED_EVENT, { detail })); } function authExpiredError(statusText: string): ApiError { - return new ApiError(401, statusText || "Unauthorized", "Your session has expired. Sign in again to continue."); + return new ApiError(401, statusText || "i18n:govoplan-core.unauthorized.740b8315", "i18n:govoplan-core.your_session_has_expired_sign_in_again_to_contin.4cc6a1b3"); } export async function apiFetch(settings: ApiSettings, path: string, init?: RequestInit): Promise { @@ -214,17 +282,27 @@ export async function apiFetch(settings: ApiSettings, path: string, init?: Re } if (isUnsafeMethod(method)) { - safeRequestGeneration += 1; - inFlightSafeRequests.clear(); - recentSafeRequests.clear(); + clearSafeRequestCaches(); } const url = apiUrl(settings, path); - const fetchInit = { ...init, headers, credentials: init?.credentials ?? "include" }; + const reusableSafeRequest = canReuseSafeRequest(method, init); + const cacheKey = reusableSafeRequest ? safeRequestKey(url, method, headers, init) : null; async function runFetch(): Promise { + const fetchHeaders = new Headers(headers); + const conditional = cacheKey ? conditionalSafeResponse(cacheKey) : undefined; + if (conditional && !fetchHeaders.has("If-None-Match")) { + fetchHeaders.set("If-None-Match", conditional.etag); + } + const fetchInit = { ...init, headers: fetchHeaders, credentials: init?.credentials ?? "include" }; const response = await fetch(url, fetchInit); + if (response.status === 304 && cacheKey && conditional) { + rememberSafeResponse(cacheKey, conditional.value); + return conditional.value as T; + } + if (!response.ok) { const text = await response.text(); if (response.status === 401 && shouldNotifyAuthRequired(path)) { @@ -235,22 +313,31 @@ export async function apiFetch(settings: ApiSettings, path: string, init?: Re } if (response.status === 204) { + if (cacheKey) conditionalSafeRequests.delete(cacheKey); return undefined as T; } const contentType = response.headers.get("content-type") || ""; + let value: T; if (!contentType.includes("application/json")) { - return (await response.text()) as T; + value = (await response.text()) as T; + } else { + value = (await response.json()) as T; } - return (await response.json()) as T; + const etag = response.headers.get("etag"); + if (cacheKey && etag) { + rememberConditionalSafeResponse(cacheKey, etag, value); + } else if (cacheKey) { + conditionalSafeRequests.delete(cacheKey); + } + return value; } - if (!canReuseSafeRequest(method, init)) { + if (!reusableSafeRequest || !cacheKey) { return runFetch(); } - const cacheKey = safeRequestKey(url, method, headers, init); const requestGeneration = safeRequestGeneration; const recent = recentSafeResponse(cacheKey); if (recent !== undefined) { @@ -262,18 +349,18 @@ export async function apiFetch(settings: ApiSettings, path: string, init?: Re return existing as Promise; } - const request = runFetch() - .then((value) => { - if (requestGeneration === safeRequestGeneration) { - rememberSafeResponse(cacheKey, value); - } - return value; - }) - .finally(() => { - if (inFlightSafeRequests.get(cacheKey) === request) { - inFlightSafeRequests.delete(cacheKey); - } - }); + const request = runFetch(). + then((value) => { + if (requestGeneration === safeRequestGeneration) { + rememberSafeResponse(cacheKey, value); + } + return value; + }). + finally(() => { + if (inFlightSafeRequests.get(cacheKey) === request) { + inFlightSafeRequests.delete(cacheKey); + } + }); inFlightSafeRequests.set(cacheKey, request); return request; } diff --git a/webui/src/api/platform.ts b/webui/src/api/platform.ts index 0c8336a..33738f4 100644 --- a/webui/src/api/platform.ts +++ b/webui/src/api/platform.ts @@ -9,6 +9,15 @@ export type PlatformStatusResponse = { enabled: boolean; message?: string | null; }; + i18n?: { + available_languages: Array<{ + code: string; + label: string; + native_label?: string | null; + }>; + enabled_languages: string[]; + default_language: string; + }; }; export type PlatformPermission = { diff --git a/webui/src/components/AccessBoundary.tsx b/webui/src/components/AccessBoundary.tsx index 7c7ff8b..56aebb3 100644 --- a/webui/src/components/AccessBoundary.tsx +++ b/webui/src/components/AccessBoundary.tsx @@ -39,22 +39,22 @@ export function ResourceAccessBoundary({ probe, resetKey, fallback, children }: let cancelled = false; setState("checking"); setMessage(""); - probe() - .then(() => { if (!cancelled) setState("allowed"); }) - .catch((error) => { - if (cancelled) return; - if (isApiError(error, 403, 404)) { - setState("denied"); - return; - } - setMessage(error instanceof Error ? error.message : String(error)); - setState("error"); - }); - return () => { cancelled = true; }; + probe(). + then(() => {if (!cancelled) setState("allowed");}). + catch((error) => { + if (cancelled) return; + if (isApiError(error, 403, 404)) { + setState("denied"); + return; + } + setMessage(error instanceof Error ? error.message : String(error)); + setState("error"); + }); + return () => {cancelled = true;}; }, [probe, resetKey]); if (state === "denied") return ; - if (state === "error") return
{message || "The resource could not be loaded."}
; - if (state === "checking") return

Checking access…

; + if (state === "error") return
{message || "i18n:govoplan-core.the_resource_could_not_be_loaded.0d1b6cbf"}
; + if (state === "checking") return

i18n:govoplan-core.checking_access.c5108c02

; return <>{children}; -} +} \ No newline at end of file diff --git a/webui/src/components/ActionBlockerHint.tsx b/webui/src/components/ActionBlockerHint.tsx new file mode 100644 index 0000000..b1a72f1 --- /dev/null +++ b/webui/src/components/ActionBlockerHint.tsx @@ -0,0 +1,64 @@ +import { AlertTriangle, Info } from "lucide-react"; +import type { ReactNode } from "react"; +import AdvancedOptionsPanel from "./AdvancedOptionsPanel"; + +export type ActionBlockerReason = { + summary: ReactNode; + details?: ReactNode; + requiredAction?: ReactNode; + actor?: ReactNode; + target?: ReactNode; + technicalDetails?: ReactNode; +}; + +type ActionBlockerHintProps = { + reason: ActionBlockerReason; + tone?: "info" | "warning" | "danger"; + className?: string; +}; + +function joinClasses(...classes: Array) { + return classes.filter(Boolean).join(" "); +} + +export default function ActionBlockerHint({ reason, tone = "warning", className = "" }: ActionBlockerHintProps) { + const Icon = tone === "info" ? Info : AlertTriangle; + const hasActionRows = Boolean(reason.requiredAction || reason.actor || reason.target); + + return ( +
+
+ ); +} diff --git a/webui/src/components/AdvancedOptionsPanel.tsx b/webui/src/components/AdvancedOptionsPanel.tsx new file mode 100644 index 0000000..9973291 --- /dev/null +++ b/webui/src/components/AdvancedOptionsPanel.tsx @@ -0,0 +1,33 @@ +import { ChevronDown } from "lucide-react"; +import type { ReactNode } from "react"; + +type AdvancedOptionsPanelProps = { + title?: ReactNode; + summary?: ReactNode; + children: ReactNode; + defaultOpen?: boolean; + className?: string; +}; + +function joinClasses(...classes: Array) { + return classes.filter(Boolean).join(" "); +} + +export default function AdvancedOptionsPanel({ + title = "Advanced options", + summary, + children, + defaultOpen = false, + className = "" +}: AdvancedOptionsPanelProps) { + return ( +
+ + {title} + + {summary &&

{summary}

} +
{children}
+
+ ); +} diff --git a/webui/src/components/Card.tsx b/webui/src/components/Card.tsx index 2da2529..545975d 100644 --- a/webui/src/components/Card.tsx +++ b/webui/src/components/Card.tsx @@ -1,5 +1,6 @@ import { useEffect, useState, type ReactNode } from "react"; import { ChevronDown } from "lucide-react"; +import { usePlatformLanguage } from "../i18n/LanguageContext"; type CardProps = { title?: ReactNode; @@ -33,17 +34,19 @@ function writeCollapseState(storageKey: string | null, collapsed: boolean): void try { window.localStorage.setItem(storageKey, collapsed ? "1" : "0"); } catch { + // localStorage may be unavailable in private or restricted contexts. - } -} + }} export default function Card({ title, children, actions, collapsible = false, collapseKey, persistCollapse = true }: CardProps) { + const { translateText } = usePlatformLanguage(); const storageKey = resolveCollapseStorageKey(collapsible, persistCollapse, collapseKey, title); const [collapseState, setCollapseState] = useState(() => ({ storageKey, collapsed: readCollapseState(storageKey) })); const collapsed = collapseState.storageKey === storageKey ? collapseState.collapsed : readCollapseState(storageKey); const hasHeader = Boolean(title || actions || collapsible); const body =
{children}
; const shouldRenderBody = !collapsible || !collapsed; + const collapseLabel = translateText(collapsed ? "i18n:govoplan-core.show_content.0528d8d2" : "i18n:govoplan-core.show_header_only.24afefca"); useEffect(() => { setCollapseState({ storageKey, collapsed: readCollapseState(storageKey) }); @@ -57,29 +60,29 @@ export default function Card({ title, children, actions, collapsible = false, co return (
- {hasHeader && ( -
- {title && (typeof title === "string" ?

{title}

:
{title}
)} - {(actions || collapsible) && ( -
+ {hasHeader && +
+ {title && (typeof title === "string" ?

{translateText(title)}

:
{title}
)} + {(actions || collapsible) && +
{actions} - {collapsible && ( - - )} + }
- )} + }
- )} + } {shouldRenderBody && (collapsible ?
{body}
: body)} -
- ); -} + ); + +} \ No newline at end of file diff --git a/webui/src/components/ColorPickerField.tsx b/webui/src/components/ColorPickerField.tsx new file mode 100644 index 0000000..793a38f --- /dev/null +++ b/webui/src/components/ColorPickerField.tsx @@ -0,0 +1,105 @@ +import { i18nMessage } from "../i18n/LanguageContext";import { Palette } from "lucide-react"; +import { useEffect, useRef, useState, type InputHTMLAttributes } from "react"; + +type ColorPickerFieldProps = Omit, "type" | "value" | "onChange"> & { + value: string; + onChange: (value: string) => void; + swatches?: string[]; +}; + +const DEFAULT_SWATCHES = [ +"#4f8cff", "#38a169", "#d69e2e", "#e53e3e", "#805ad5", "#319795", +"#dd6b20", "#718096", "#1f2937", "#f59e0b", "#10b981", "#ef4444"]; + + +const HEX_PATTERN = /^#[0-9a-fA-F]{6}$/; + +function normalizeColor(value: string): string { + const trimmed = value.trim(); + if (/^[0-9a-fA-F]{6}$/.test(trimmed)) return `#${trimmed}`; + return trimmed; +} + +function useOutsideClose(open: boolean, onClose: () => void) { + const ref = useRef(null); + useEffect(() => { + if (!open) return; + function handlePointerDown(event: PointerEvent) { + if (!ref.current || ref.current.contains(event.target as Node)) return; + onClose(); + } + function handleKeyDown(event: KeyboardEvent) { + if (event.key === "Escape") onClose(); + } + document.addEventListener("pointerdown", handlePointerDown); + document.addEventListener("keydown", handleKeyDown); + return () => { + document.removeEventListener("pointerdown", handlePointerDown); + document.removeEventListener("keydown", handleKeyDown); + }; + }, [open, onClose]); + return ref; +} + +export default function ColorPickerField({ + value, + onChange, + swatches = DEFAULT_SWATCHES, + disabled, + className = "", + placeholder = "#4f8cff", + ...props +}: ColorPickerFieldProps) { + const [open, setOpen] = useState(false); + const rootRef = useOutsideClose(open, () => setOpen(false)); + const inputRef = useRef(null); + const normalizedValue = normalizeColor(value || ""); + const previewColor = HEX_PATTERN.test(normalizedValue) ? normalizedValue : "transparent"; + + useEffect(() => { + const input = inputRef.current; + if (!input) return; + input.setCustomValidity(normalizedValue && !HEX_PATTERN.test(normalizedValue) ? "i18n:govoplan-core.use_a_six_digit_hex_color_for_example_4f8cff.d74f4ad2" : ""); + }, [normalizedValue]); + + function selectColor(nextColor: string) { + onChange(nextColor); + setOpen(false); + } + + return ( +
+
); + +} \ No newline at end of file diff --git a/webui/src/components/ConfirmDialog.tsx b/webui/src/components/ConfirmDialog.tsx index 8efe008..80b8d3c 100644 --- a/webui/src/components/ConfirmDialog.tsx +++ b/webui/src/components/ConfirmDialog.tsx @@ -1,5 +1,6 @@ import Button from "./Button"; import Dialog from "./Dialog"; +import { usePlatformLanguage } from "../i18n/LanguageContext"; export type ConfirmDialogTone = "default" | "danger"; @@ -19,13 +20,14 @@ export default function ConfirmDialog({ open, title, message, - confirmLabel = "Confirm", - cancelLabel = "Cancel", + confirmLabel = "i18n:govoplan-core.confirm.04a21221", + cancelLabel = "i18n:govoplan-core.cancel.77dfd213", tone = "default", busy = false, onConfirm, onCancel }: ConfirmDialogProps) { + const { translateText } = usePlatformLanguage(); return ( - - + footer={ + <> + + - )} - > -

{message}

- - ); + }> + +

{translateText(message)}

+ ); + } diff --git a/webui/src/components/ConnectionTree.tsx b/webui/src/components/ConnectionTree.tsx new file mode 100644 index 0000000..d1b1ca3 --- /dev/null +++ b/webui/src/components/ConnectionTree.tsx @@ -0,0 +1,84 @@ +import type { CSSProperties, ReactNode } from "react"; + +export type ConnectionTreeColumn = { + id: string; + header: ReactNode; + width?: string; + align?: "left" | "center" | "right"; + render: (row: T, depth: number) => ReactNode; +}; + +export type ConnectionTreeProps = { + rows: T[]; + columns: ConnectionTreeColumn[]; + getRowKey: (row: T) => string; + getChildren?: (row: T) => T[]; + renderActions?: (row: T, depth: number) => ReactNode; + emptyText?: ReactNode; + className?: string; + rowClassName?: (row: T, depth: number) => string; +}; + +type FlatRow = { + row: T; + depth: number; +}; + +export default function ConnectionTree({ + rows, + columns, + getRowKey, + getChildren, + renderActions, + emptyText = "i18n:govoplan-core.no_entries_configured.48e7b97c", + className = "", + rowClassName +}: ConnectionTreeProps) { + const flatRows = flattenRows(rows, getChildren); + const gridTemplateColumns = `${columns.map((column) => column.width || "minmax(0, 1fr)").join(" ")} ${renderActions ? "var(--connection-tree-actions-column, 196px)" : ""}`.trim(); + + return ( +
+
+ {columns.map((column) => +
+ {column.header} +
+ )} + {renderActions &&
i18n:govoplan-core.actions.c3cd636a
} +
+ {flatRows.length === 0 &&
{emptyText}
} + {flatRows.map(({ row, depth }) => { + const className = ["connection-tree-row", depth > 0 ? "is-child" : "is-parent", rowClassName?.(row, depth) || ""].filter(Boolean).join(" "); + return ( +
+ {columns.map((column, index) => +
+ {index === 0 ? +
+ {column.render(row, depth)} +
: + column.render(row, depth) + } +
+ )} + {renderActions &&
{renderActions(row, depth)}
} +
); + + })} +
); + +} + +function flattenRows(rows: T[], getChildren?: (row: T) => T[]): FlatRow[] { + const result: FlatRow[] = []; + for (const row of rows) { + result.push({ row, depth: 0 }); + for (const child of getChildren?.(row) ?? []) { + result.push({ row: child, depth: 1 }); + } + } + return result; +} diff --git a/webui/src/components/DateTimeField.tsx b/webui/src/components/DateTimeField.tsx new file mode 100644 index 0000000..953e992 --- /dev/null +++ b/webui/src/components/DateTimeField.tsx @@ -0,0 +1,231 @@ +import { CalendarDays, ChevronLeft, ChevronRight, Clock } from "lucide-react"; +import { useEffect, useMemo, useRef, useState, type InputHTMLAttributes } from "react"; + +type BaseProps = Omit, "type" | "value" | "onChange" | "min" | "max"> & { + value: string; + onChange: (value: string) => void; + min?: string; + max?: string; +}; + +const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/; +const TIME_PATTERN = /^\d{2}:\d{2}$/; + +function pad(value: number): string { + return String(value).padStart(2, "0"); +} + +function dateString(date: Date): string { + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +} + +function parseDate(value: string): Date | null { + if (!DATE_PATTERN.test(value)) return null; + const [year, month, day] = value.split("-").map(Number); + const date = new Date(year, month - 1, day); + if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) return null; + return date; +} + +function monthLabel(date: Date): string { + return date.toLocaleDateString(undefined, { month: "long", year: "numeric" }); +} + +function clampDateValue(value: string, min?: string, max?: string): string { + if (min && value < min) return min; + if (max && value > max) return max; + return value; +} + +function datePartsFromDateTime(value: string): {date: string;time: string;} { + if (!value) return { date: "", time: "" }; + const [date = "", time = ""] = value.split("T", 2); + return { date, time: time.slice(0, 5) }; +} + +function combineDateTime(date: string, time: string): string { + if (!date && !time) return ""; + return `${date || dateString(new Date())}T${time || "00:00"}`; +} + +function useOutsideClose(open: boolean, onClose: () => void) { + const ref = useRef(null); + useEffect(() => { + if (!open) return; + function handlePointerDown(event: PointerEvent) { + if (!ref.current || ref.current.contains(event.target as Node)) return; + onClose(); + } + function handleKeyDown(event: KeyboardEvent) { + if (event.key === "Escape") onClose(); + } + document.addEventListener("pointerdown", handlePointerDown); + document.addEventListener("keydown", handleKeyDown); + return () => { + document.removeEventListener("pointerdown", handlePointerDown); + document.removeEventListener("keydown", handleKeyDown); + }; + }, [open, onClose]); + return ref; +} + +export function DateField({ value, onChange, min, max, disabled, className = "", placeholder = "i18n:govoplan-core.yyyy_mm_dd.d3f8f7b8", ...props }: BaseProps) { + const selectedDate = parseDate(value); + const [open, setOpen] = useState(false); + const [visibleMonth, setVisibleMonth] = useState(() => selectedDate ?? new Date()); + const rootRef = useOutsideClose(open, () => setOpen(false)); + const inputRef = useRef(null); + + useEffect(() => { + if (selectedDate) setVisibleMonth(new Date(selectedDate.getFullYear(), selectedDate.getMonth(), 1)); + }, [value]); // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + const input = inputRef.current; + if (!input) return; + let message = ""; + if (value && !parseDate(value)) message = "i18n:govoplan-core.use_yyyy_mm_dd.406d2e4d";else + if (value && min && value < min) message = `Use ${min} or later.`;else + if (value && max && value > max) message = `Use ${max} or earlier.`; + input.setCustomValidity(message); + }, [value, min, max]); + + const weeks = useMemo(() => { + const first = new Date(visibleMonth.getFullYear(), visibleMonth.getMonth(), 1); + const start = new Date(first); + const weekday = (first.getDay() + 6) % 7; + start.setDate(first.getDate() - weekday); + return Array.from({ length: 42 }, (_unused, index) => { + const date = new Date(start); + date.setDate(start.getDate() + index); + return date; + }); + }, [visibleMonth]); + + function selectDate(nextDate: Date) { + const nextValue = clampDateValue(dateString(nextDate), min, max); + onChange(nextValue); + setOpen(false); + } + + function moveMonth(delta: number) { + setVisibleMonth((current) => new Date(current.getFullYear(), current.getMonth() + delta, 1)); + } + + return ( +
+ onChange(event.target.value)} /> + + + {open && +
+
+ + {monthLabel(visibleMonth)} + +
+ +
+ {weeks.map((day) => { + const dayValue = dateString(day); + const inMonth = day.getMonth() === visibleMonth.getMonth(); + const isSelected = dayValue === value; + const unavailable = Boolean(min && dayValue < min || max && dayValue > max); + return ( + ); + + })} +
+
+ } +
); + +} + +export function TimeField({ value, onChange, min, max, className = "", placeholder = "i18n:govoplan-core.hh_mm.a4c7ee9b", ...props }: BaseProps) { + const inputRef = useRef(null); + useEffect(() => { + const input = inputRef.current; + if (!input) return; + let message = ""; + if (value && !TIME_PATTERN.test(value)) message = "i18n:govoplan-core.use_hh_mm.e995be3f";else + if (value && min && value < min) message = `Use ${min} or later.`;else + if (value && max && value > max) message = `Use ${max} or earlier.`; + input.setCustomValidity(message); + }, [value, min, max]); + + return ( +
+ onChange(event.target.value)} /> + +
); + +} + +export function DateTimeField({ value, onChange, min, max, disabled, className = "", ...props }: BaseProps) { + const parts = datePartsFromDateTime(value); + const minParts = datePartsFromDateTime(min || ""); + const maxParts = datePartsFromDateTime(max || ""); + + function changeDate(nextDate: string) { + onChange(combineDateTime(nextDate, parts.time)); + } + + function changeTime(nextTime: string) { + onChange(combineDateTime(parts.date, nextTime)); + } + + return ( +
+ + + + +
); + +} + +export default DateField; \ No newline at end of file diff --git a/webui/src/components/Dialog.tsx b/webui/src/components/Dialog.tsx index cc16866..a38e99f 100644 --- a/webui/src/components/Dialog.tsx +++ b/webui/src/components/Dialog.tsx @@ -1,4 +1,5 @@ import { useEffect, useId, type ReactNode } from "react"; +import { translateReactNode, usePlatformLanguage } from "../i18n/LanguageContext"; export type DialogProps = { open: boolean; @@ -31,7 +32,7 @@ export default function Dialog({ footer, role = "dialog", ariaDescribedBy, - closeLabel = "Close", + closeLabel = "i18n:govoplan-core.close.bbfa773e", closeOnBackdrop = true, showCloseButton = true, closeDisabled = false, @@ -45,6 +46,11 @@ export default function Dialog({ }: DialogProps) { const titleId = useId(); const canClose = Boolean(onClose) && !closeDisabled; + const { translateText } = usePlatformLanguage(); + const renderedTitle = translateReactNode(title, translateText); + const renderedChildren = translateReactNode(children, translateText); + const renderedFooter = translateReactNode(footer, translateText); + const translatedCloseLabel = translateText(closeLabel); useEffect(() => { if (!open || !canClose) return undefined; @@ -75,21 +81,21 @@ export default function Dialog({ aria-describedby={ariaDescribedBy} >
-

{title}

+

{renderedTitle}

{showCloseButton && onClose && ( )}
-
{children}
- {footer &&
{footer}
} +
{renderedChildren}
+ {footer &&
{renderedFooter}
} ); diff --git a/webui/src/components/DismissibleAlert.tsx b/webui/src/components/DismissibleAlert.tsx index b0676d0..0312f75 100644 --- a/webui/src/components/DismissibleAlert.tsx +++ b/webui/src/components/DismissibleAlert.tsx @@ -1,6 +1,7 @@ import { useEffect, useState, type ReactNode } from "react"; import { createPortal } from "react-dom"; import { X } from "lucide-react"; +import { translateReactNode, usePlatformLanguage } from "../i18n/LanguageContext"; type AlertTone = "success" | "info" | "warning" | "danger"; @@ -17,20 +18,24 @@ type DismissibleAlertProps = { let floatingAlertRoot: HTMLElement | null = null; -function getFloatingAlertRoot(): HTMLElement | null { +function getFloatingAlertRoot(ariaLabel: string): HTMLElement | null { if (typeof document === "undefined") return null; - if (floatingAlertRoot?.isConnected) return floatingAlertRoot; + if (floatingAlertRoot?.isConnected) { + floatingAlertRoot.setAttribute("aria-label", ariaLabel); + return floatingAlertRoot; + } const existing = document.getElementById("app-floating-alerts"); if (existing) { floatingAlertRoot = existing; + floatingAlertRoot.setAttribute("aria-label", ariaLabel); return floatingAlertRoot; } floatingAlertRoot = document.createElement("div"); floatingAlertRoot.id = "app-floating-alerts"; floatingAlertRoot.className = "alert-floating-stack"; - floatingAlertRoot.setAttribute("aria-label", "Application notices"); + floatingAlertRoot.setAttribute("aria-label", ariaLabel); document.body.appendChild(floatingAlertRoot); return floatingAlertRoot; } @@ -68,6 +73,7 @@ export default function DismissibleAlert({ resetKey, dismissStorageKey }: DismissibleAlertProps) { + const { translateText } = usePlatformLanguage(); const storageKey = resolveDismissStorageKey(dismissStorageKey, resetKey); const [alertState, setAlertState] = useState(() => ({ storageKey, visible: !readDismissed(storageKey) })); const visible = alertState.storageKey === storageKey ? alertState.visible : !readDismissed(storageKey); @@ -77,6 +83,8 @@ export default function DismissibleAlert({ }, [storageKey, resetKey, children]); if (!visible) return null; + const renderedChildren = translateReactNode(children, translateText); + const floatingLabel = translateText("i18n:govoplan-core.application_notices"); function dismiss() { writeDismissed(storageKey); @@ -90,9 +98,9 @@ export default function DismissibleAlert({ role={role} aria-live={role === "alert" ? "assertive" : "polite"} > -
{children}
+
{renderedChildren}
{dismissible && ( - )} @@ -100,6 +108,6 @@ export default function DismissibleAlert({ ); if (!floating) return alert; - const root = getFloatingAlertRoot(); + const root = getFloatingAlertRoot(floatingLabel); return root ? createPortal(alert, root) : alert; } diff --git a/webui/src/components/ExplorerTree.tsx b/webui/src/components/ExplorerTree.tsx index bc0979c..be2290b 100644 --- a/webui/src/components/ExplorerTree.tsx +++ b/webui/src/components/ExplorerTree.tsx @@ -1,5 +1,6 @@ import { Folder, FolderOpen } from "lucide-react"; import type { CSSProperties, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, ReactNode } from "react"; +import { i18nMessage } from "../i18n/LanguageContext"; export type ExplorerTreeNodeContext = { depth: number; @@ -9,7 +10,7 @@ export type ExplorerTreeNodeContext = { disabled: boolean; }; -type ExplorerTreeProps = { +export type ExplorerTreeProps = { nodes: T[]; getNodeId: (node: T) => string; getNodeLabel: (node: T) => string; @@ -21,6 +22,11 @@ type ExplorerTreeProps = { disabled?: boolean; depth?: number; className?: string; + childrenBaseClassName?: string; + nodeContainerClassName?: string; + nodeWrapBaseClassName?: string; + toggleBaseClassName?: string; + nodeButtonBaseClassName?: string; renderToggleIcon?: (node: T, context: ExplorerTreeNodeContext) => ReactNode; renderNodeContent?: (node: T, context: ExplorerTreeNodeContext) => ReactNode; getNodeWrapClassName?: (node: T, context: ExplorerTreeNodeContext) => string | undefined; @@ -47,6 +53,11 @@ export default function ExplorerTree({ disabled = false, depth = 1, className = "", + childrenBaseClassName = "explorer-tree-children file-tree-children", + nodeContainerClassName = "", + nodeWrapBaseClassName = "explorer-tree-node-wrap file-tree-node-wrap", + toggleBaseClassName = "explorer-tree-toggle file-tree-toggle", + nodeButtonBaseClassName = "explorer-tree-node file-tree-node", renderToggleIcon, renderNodeContent, getNodeWrapClassName, @@ -63,7 +74,7 @@ export default function ExplorerTree({ if (nodes.length === 0) return null; return ( -
+
{nodes.map((node) => { const nodeId = getNodeId(node); const children = getNodeChildren(node); @@ -74,14 +85,13 @@ export default function ExplorerTree({ const draggable = getNodeDraggable?.(node, context) ?? false; return ( -
+
onContextMenu(event, node, context) : undefined} @@ -89,59 +99,64 @@ export default function ExplorerTree({ onDragEnd={draggable && onDragEnd ? (event) => onDragEnd(event, node, context) : undefined} onDragOver={onDragOver ? (event) => onDragOver(event, node, context) : undefined} onDragLeave={onDragLeave ? (event) => onDragLeave(event, node, context) : undefined} - onDrop={onDrop ? (event) => onDrop(event, node, context) : undefined} - > + onDrop={onDrop ? (event) => onDrop(event, node, context) : undefined}> +
- {hasChildren && expanded && ( - - )} -
- ); + {hasChildren && expanded && + + + } +
); + })} -
- ); +
); + } diff --git a/webui/src/components/FileDropZone.tsx b/webui/src/components/FileDropZone.tsx index 4b403da..3a27d70 100644 --- a/webui/src/components/FileDropZone.tsx +++ b/webui/src/components/FileDropZone.tsx @@ -1,4 +1,4 @@ -import { useRef, useState, type CSSProperties, type ReactNode } from "react"; +import { i18nMessage } from "../i18n/LanguageContext";import { useRef, useState, type CSSProperties, type ReactNode } from "react"; import { UploadCloud } from "lucide-react"; export type FileDropZoneProps = { @@ -23,13 +23,13 @@ export default function FileDropZone({ disabled = false, busy = false, progress = null, - label = "Drop files here", - actionLabel = "or click to select files", - busyLabel = "Uploading files", + label = "i18n:govoplan-core.drop_files_here.77348907", + actionLabel = "i18n:govoplan-core.or_click_to_select_files.91b05dc1", + busyLabel = "i18n:govoplan-core.uploading_files.6536791d", progressLabel, note, className = "", - inputLabel = "Drop files here or click to select files", + inputLabel = "i18n:govoplan-core.drop_files_here_or_click_to_select_files.7eda8608", onFiles }: FileDropZoneProps) { const inputRef = useRef(null); @@ -79,25 +79,25 @@ export default function FileDropZone({ event.preventDefault(); setDragActive(false); if (!interactionDisabled) void handleFiles(event.dataTransfer.files); - }} - > - {showProgress ? ( - - {roundedProgress === null ? "" : `${roundedProgress}%`} - - ) : ( -