diff --git a/.gitea/ISSUE_TEMPLATE/bug_report.md b/.gitea/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..cf14bd1 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,35 @@ +--- +name: "Bug" +about: "Report a reproducible defect, regression, or incorrect behavior" +title: "[Bug] " +labels: + - type/bug + - status/triage + - module/core +--- + +## Scope + +- Repository: +- Area/module: +- Affected version or commit: + +## Behavior + +Expected: + +Actual: + +## Reproduction + +1. +2. +3. + +## Evidence + +Logs, screenshots, traces, or failing test output: + +## Verification Target + +Command or workflow that should pass when fixed: diff --git a/.gitea/ISSUE_TEMPLATE/config.yaml b/.gitea/ISSUE_TEMPLATE/config.yaml new file mode 100644 index 0000000..3ba13e0 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/config.yaml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.gitea/ISSUE_TEMPLATE/docs_workflow.md b/.gitea/ISSUE_TEMPLATE/docs_workflow.md new file mode 100644 index 0000000..a3b0cca --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/docs_workflow.md @@ -0,0 +1,27 @@ +--- +name: "Docs / workflow" +about: "Request documentation, process, or developer workflow changes" +title: "[Docs] " +labels: + - type/docs + - status/triage + - module/core + - area/docs +--- + +## Scope + +- Repository: +- Document or workflow: + +## Current State + +What is missing, unclear, duplicated, or stale? + +## Desired State + +What should the docs or workflow make clear? + +## Verification Target + +How should this be checked? diff --git a/.gitea/ISSUE_TEMPLATE/feature_request.md b/.gitea/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..b286a18 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: "Feature" +about: "Propose new user-visible behavior or platform capability" +title: "[Feature] " +labels: + - type/feature + - status/triage + - module/core +--- + +## Problem + +What user, operator, or developer problem should this solve? + +## Proposed Capability + +What should exist when this is done? + +## Ownership + +- Owning repository: +- Related module repositories: +- Extension point or integration boundary: + +## Acceptance Criteria + +- [ ] +- [ ] + +## Verification Target + +Command, scenario, or UI flow that should prove completion: diff --git a/.gitea/ISSUE_TEMPLATE/task.md b/.gitea/ISSUE_TEMPLATE/task.md new file mode 100644 index 0000000..30ea442 --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/task.md @@ -0,0 +1,28 @@ +--- +name: "Task" +about: "Track implementation, maintenance, or migration work" +title: "[Task] " +labels: + - type/task + - status/triage + - module/core +--- + +## Objective + +What needs to be completed? + +## Scope + +- Owning repository: +- In-scope: +- Out-of-scope: + +## Checklist + +- [ ] +- [ ] + +## Verification Target + +Command or manual check: diff --git a/.gitea/ISSUE_TEMPLATE/tech_debt.md b/.gitea/ISSUE_TEMPLATE/tech_debt.md new file mode 100644 index 0000000..09eec4f --- /dev/null +++ b/.gitea/ISSUE_TEMPLATE/tech_debt.md @@ -0,0 +1,25 @@ +--- +name: "Tech debt" +about: "Track cleanup, refactoring, risk reduction, or deferred engineering work" +title: "[Debt] " +labels: + - type/debt + - status/triage + - module/core +--- + +## Current Cost + +What does this make harder, riskier, slower, or more fragile? + +## Desired Shape + +What should the code, tests, or architecture look like afterwards? + +## Constraints + +What behavior, compatibility, or module boundary must be preserved? + +## Verification Target + +Focused checks that should pass: diff --git a/.gitea/PULL_REQUEST_TEMPLATE.md b/.gitea/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..1984736 --- /dev/null +++ b/.gitea/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,15 @@ +## Issue + +Closes # + +## Summary + +- + +## Verification + +- + +## Notes + +Follow-up issues: diff --git a/AGENTS.md b/AGENTS.md index 9b35991..056e63c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,7 @@ This repository is the platform runner and shared core for GovOPlaN. It owns the Sibling module repositories usually used with this repo: +- `/mnt/DATA/git/govoplan-access` - `/mnt/DATA/git/govoplan-files` - `/mnt/DATA/git/govoplan-mail` - `/mnt/DATA/git/govoplan-campaign` @@ -44,5 +45,6 @@ cd /mnt/DATA/git/govoplan-core - Prefer `rg`, `sed`, and targeted tests over broad recursive scans or full builds. - Avoid DataGrid changes unless explicitly requested; it is intentionally brittle and has known deferred work. - Do not add module-to-module imports for optional integrations. Use core registry/capability/module metadata paths. +- Treat Gitea issues as the canonical backlog and state log. Treat Gitea wiki pages as durable project context mirrored from repository and product docs. Use `docs/GITEA_ISSUES.md` for labels, templates, TODO import, wiki sync, and Codex issue updates. - Do not keep generated WebUI test folders in git status; they should be ignored and removable. - Do not start persistent dev servers unless the user asks. diff --git a/README.md b/README.md index 83eeb8c..d2fe4a9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ cd /mnt/DATA/git/govoplan-core ./.venv/bin/python -m pip install -r requirements-dev.txt ``` -Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `access,campaigns,files,mail`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation. +Run the platform server from core through the module-aware development runner. The default config reads `ENABLED_MODULES` and discovers installed module entry points. Local development defaults to `tenancy,access,admin,policy,audit,campaigns,files,mail`; set `ENABLED_MODULES` explicitly when testing a smaller module permutation. ```bash cd /mnt/DATA/git/govoplan-core @@ -70,7 +70,7 @@ cd /mnt/DATA/git/govoplan-core The smoke mode prints the effective config, runtime root, database URL, modules, reload state, and bootstrap decision, then creates the ASGI app and runs startup once. -`requirements-dev.txt` links local `govoplan-files`, `govoplan-mail`, and `govoplan-campaign` checkouts for development. `requirements-release.txt` installs those modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md). +`requirements-dev.txt` links local GovOPlaN module checkouts for development. `requirements-release.txt` installs the packaged modules from tagged git refs for release builds. See [RELEASE_DEPENDENCIES.md](docs/RELEASE_DEPENDENCIES.md). ## WebUI development diff --git a/alembic/env.py b/alembic/env.py index 3f98c16..efe5273 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -5,10 +5,10 @@ from logging.config import fileConfig from alembic import context from sqlalchemy import engine_from_config, pool -from govoplan_core.access.db import models as access_models # noqa: F401 - populate access metadata +from govoplan_access.backend.db import models as access_models # noqa: F401 - populate access metadata +from govoplan_core.admin import models as core_admin_models # noqa: F401 - populate core admin metadata from govoplan_core.core.migrations import migration_metadata_plan from govoplan_core.db.base import Base -from govoplan_core.db import models # noqa: F401 - populate core metadata from govoplan_core.server.default_config import get_server_config from govoplan_core.server.registry import build_platform_registry from govoplan_core.settings import settings @@ -23,9 +23,11 @@ if config.config_file_name is not None: def _target_metadata(): server_config = get_server_config() + enabled_modules = config.attributes.get("enabled_modules", server_config.enabled_modules) + manifest_factories = config.attributes.get("manifest_factories", server_config.manifest_factories) registry = build_platform_registry( - server_config.enabled_modules, - manifest_factories=server_config.manifest_factories, + enabled_modules, + manifest_factories=manifest_factories, ) plan = migration_metadata_plan(registry, extra_metadata=(Base.metadata,)) return tuple(dict.fromkeys(plan.metadata)) diff --git a/docs/ACCESS_EXTRACTION_PLAN.md b/docs/ACCESS_EXTRACTION_PLAN.md new file mode 100644 index 0000000..dec2ec0 --- /dev/null +++ b/docs/ACCESS_EXTRACTION_PLAN.md @@ -0,0 +1,456 @@ +# GovOPlaN Access Extraction Plan + +> Backlog state migrated to Gitea issues on 2026-07-06. Keep this document as +> durable extraction context and architecture notes. Track active tasks, +> decisions, blockers, and implementation state in `add-ideas/govoplan-core` +> issues with `module/access` and `source/backlog-import`. + +This plan describes how to extract access, authentication, RBAC, and related +administration behavior from the current compatibility core into a dedicated +`govoplan-access` platform module. + +The goal is not to make access optional in every real deployment. The goal is to +make ownership explicit: the kernel composes modules and exposes contracts; +`govoplan-access` owns identity, sessions, API keys, roles, groups, and access +administration. + +## Current Inventory + +### Existing Access Module Seed + +`govoplan-access` contains the extracted module-shaped seed package under +`src/govoplan_access/backend`. `govoplan-core` keeps compatibility import shims +under `src/govoplan_core/access`: + +- `manifest.py` declares `ModuleManifest(id="access")`. +- `db/models.py` defines access-owned model candidates. +- `auth/principals.py`, `auth/roles.py`, and `auth/tokens.py` contain auth + helper concepts. +- `permissions/definitions.py`, `permissions/evaluator.py`, and + `permissions/registry.py` contain permission catalogue/evaluation concepts. +- `tenancy/datastore.py` contains early tenancy access helpers. + +This seed package is now the starting point for the access module, but it is +not yet the full live source of truth for the running product. + +### Live Compatibility Ownership In Core + +The active implementation still uses core-owned models and services: + +- `src/govoplan_core/db/models.py` + - `Account` + - `Tenant` + - `User` + - `Group` + - `Role` + - `SystemSettings` + - `GovernanceTemplate` + - `GovernanceTemplateAssignment` + - `SystemRoleAssignment` + - `UserGroupMembership` + - `UserRoleAssignment` + - `GroupRoleAssignment` + - `ApiKey` + - `AuthSession` + - `AuditLog` +- `src/govoplan_core/auth/dependencies.py` +- `src/govoplan_core/security/api_keys.py` +- `src/govoplan_core/security/permissions.py` +- `src/govoplan_core/security/sessions.py` +- `src/govoplan_core/security/passwords.py` +- `src/govoplan_core/security/secrets.py` +- `src/govoplan_core/security/module_permissions.py` +- `src/govoplan_core/admin/service.py` +- `src/govoplan_core/admin/governance.py` +- `src/govoplan_core/api/v1/auth.py` +- `src/govoplan_core/api/v1/admin.py` +- `src/govoplan_core/api/v1/admin_schemas.py` +- `src/govoplan_core/api/v1/audit.py` +- `src/govoplan_core/db/bootstrap.py` + +Core still hosts the generic login/settings shell and shared WebUI primitives. +The legacy administration page has moved to the `govoplan-access` WebUI package +and is contributed as the `/admin` route by the access module. Individual admin +panels can now be split further into access, admin, tenancy, policy, and audit +WebUI contributions without changing the core shell route wiring again. + +### Current Module Consumers + +Feature modules no longer import core auth dependency wrappers or access-owned +ORM models. Backend routers import the access-published FastAPI dependency API +from `govoplan_access.backend.auth.dependencies`; runtime cooperation uses +kernel capabilities such as `access.directory`, `campaigns.access`, +`campaigns.mailPolicyContext`, and `campaigns.deliveryTasks`. + +## Target Ownership + +### Kernel + +The kernel keeps only platform composition and stable contracts: + +- module discovery and registry validation +- route aggregation +- migration orchestration +- database engine/session lifecycle +- permission catalogue aggregation +- capability registry +- event/command envelopes +- dependency injection hooks +- health and diagnostics +- compatibility facades while modules migrate + +The kernel should not own account, tenant, role, group, session, or API-key +semantics. + +### `govoplan-access` + +`govoplan-access` owns: + +- accounts +- authentication routes +- session lifecycle +- API keys +- users +- groups and memberships +- roles and assignments +- principal resolution +- permission evaluation +- access administration routes +- access administration WebUI contributions +- access-related migrations +- access permissions and role templates + +### Later Platform Modules + +Some current access-adjacent behavior can remain in access initially, but should +have clean seams for later extraction: + +- `govoplan-tenancy`: tenants, tenant lifecycle, tenant switching, tenant + metadata, tenant settings boundaries. +- `govoplan-policy`: hierarchical policy/effective policy resolution and + provenance display contracts. +- `govoplan-audit`: audit log storage, audit routes, retention hooks, evidence + exports. +- `govoplan-admin`: generic administration shell contributions if they are not + owned by access/tenancy/policy/audit directly. + +## Kernel Contracts To Stabilize First + +Before moving live code, define contracts that feature modules can depend on +without importing access ORM models: + +- `PrincipalResolver` + - resolves the current actor from a request/session/API key. + - returns a stable DTO, not an ORM model. +- `AccessDirectory` + - resolves users, groups, memberships, and display labels by stable IDs. + - supports batch lookups for grids and policy screens. +- `TenantResolver` + - resolves current tenant context and tenant metadata by stable ID. +- `PermissionEvaluator` + - evaluates required scopes for a principal and resource. +- `ResourceAccessProvider` + - lets modules register resource ACL behavior without importing each other. +- `SecretProvider` + - encrypts/decrypts named secrets without tying consumers to access models. +- `AuditSink` + - records audit events without importing audit storage models. + +DTOs should be small and serializable: + +- `PrincipalRef` +- `AccountRef` +- `TenantRef` +- `UserRef` +- `GroupRef` +- `RoleRef` + +## Extraction Stages + +### Stage 0: Baseline Safeguards + +Status: started. + +- Document the kernel/platform module model. +- Add dependency-boundary checks. +- Add backend module permutation startup tests. +- Inventory access/auth/RBAC imports. +- Draft this extraction plan. + +Acceptance criteria: + +- `scripts/check_dependency_boundaries.py` passes. +- backend module permutation tests start every supported module combination. +- current direct imports are tracked as explicit transitional debt. + +### Stage 1: Contract Definitions + +Add kernel-owned protocols and DTOs for access-related interaction. + +Tasks: + +- Add protocol definitions under a kernel contract package. +- Add registry/capability names for access services. +- Keep existing core dependency functions as compatibility wrappers. +- Make wrappers resolve through capabilities when `govoplan-access` is present. +- Add tests for missing-capability behavior and clear error messages. + +Acceptance criteria: + +- Feature modules can receive principal/tenant/group/user references without + importing access ORM models. +- Existing routes continue to work through compatibility wrappers. + +### Stage 2: Create `govoplan-access` + +Create the new repository/package and move the access seed package into it. + +Tasks: + +- Create package metadata and entry points for `govoplan-access`. +- Move `govoplan_core/access` implementation into the new package. +- Publish `ModuleManifest(id="access")`. +- Register access permissions, role templates, routers, and migrations. +- Add compatibility imports in core where needed. +- Add release/dev dependency entries in core. + +Acceptance criteria: + +- `govoplan-core + govoplan-access` starts through normal module discovery. +- `govoplan-core` compatibility imports still work during transition. +- access manifest, migrations, and route contributions are discovered by core. + +### Stage 3: Move Live Models And Migrations + +Move active identity/access models out of `govoplan_core.db.models`. + +Current state: the stable table naming strategy is to keep legacy table names +and move SQLAlchemy class definitions under their platform owners. The old +`govoplan_core.db.models` compatibility re-export has been removed; callers +must import module-owned models or use kernel capabilities. The earlier +`access_*` candidate tables are not active metadata. + +Current table ownership: + +- `govoplan-tenancy`: `tenants` +- `govoplan-access`: `accounts`, tenant memberships in `users`, `groups`, + `roles`, role/group assignment tables, `api_keys`, and `auth_sessions` +- `govoplan-admin`: governance templates and assignments +- `govoplan-audit`: audit log +- `govoplan-core`: system settings + +Tasks: + +- [x] Decide the stable table naming strategy. +- [x] Map legacy model names to access-owned model classes. +- [x] Keep database table names where possible to avoid data migration churn. +- [x] Add Alembic migration metadata owned by the platform module owners. +- [x] Remove core model compatibility aliases after callers moved to + module-owned imports. +- [x] Update bootstrap/create-all compatibility paths. + +Acceptance criteria: + +- Existing development databases migrate without data loss. +- New databases initialize with module-owned metadata. +- Core no longer owns or re-exports live account/user/group/role/session/API-key + model definitions. + +### Stage 4: Move Auth Routes And Dependencies + +Move authentication, sessions, API keys, and principal dependencies. + +Tasks: + +- Move `/api/v1/auth/*` implementation to access. +- Move session/API-key services to access. +- Keep core route compatibility only if required by clients. +- Replace feature-module imports of core auth dependencies with the + access-published FastAPI dependency API. +- Add tests for login, session refresh, tenant selection, API-key auth, and + missing access capability failures. + +Acceptance criteria: + +- Auth behavior works through the access module. +- Feature modules do not import access internals except the published + `govoplan_access.backend.auth.dependencies` dependency API used by FastAPI + routers. +- Core can explain startup failure clearly if auth-required routes are enabled + without the access capability. + +### Stage 5: Move Admin And WebUI Contributions + +Move access administration UI/API ownership into modules. + +Tasks: + +- [x] Move users, groups, roles, API keys, and access settings pages into + `govoplan-access`. +- [ ] Move tenant-specific pages to `govoplan-tenancy` when that module exists, or + keep them temporarily in access with clear boundaries. +- [x] Move overview, system settings, and governance-template panels into + `govoplan-admin`. +- [x] Register admin navigation and admin sections through module + contributions. +- [x] Keep core shell, layout, route rendering, and generic components only. + +Acceptance criteria: + +- Core WebUI shell renders admin/access pages from module contributions. +- Core does not import access page components directly. +- Module nav and route metadata remain serializable. +- The access admin shell consumes module-owned `admin.sections` capability + contributions without importing sibling module panels. + +### Stage 6: Decouple Feature Modules + +Remove direct ORM/model imports from files, mail, and campaign. + +Tasks: + +- Replace `Group`, `Tenant`, `User`, and `UserGroupMembership` imports with + directory/capability lookups. +- Replace direct cross-module cleanup/count queries with registered providers, + events, or module-owned API contracts. +- Replace mail-profile ownership resolution with stable owner references. +- Replace campaign/file access checks with resource ACL provider contracts. + +Acceptance criteria: + +- `govoplan-files`, `govoplan-mail`, and `govoplan-campaign` do not import + access implementation modules or sibling feature modules. +- Dependency-boundary allowlist stays empty after each replacement. + +### Stage 7: Remove Compatibility Debt + +Finalize the split. + +Tasks: + +- Remove obsolete core compatibility aliases. +- Keep the dependency-boundary checker allowlist empty. +- Update release dependency docs. +- Update operator migration notes. +- Add full module permutation tests including access-present and access-absent + behavior. + +Acceptance criteria: + +- Core is a composition kernel. +- Access behavior is owned by `govoplan-access`. +- Feature modules communicate through kernel contracts, capabilities, events, + and their own APIs. + +## Immediate Backlog + +- [x] Document kernel/platform module model. +- [x] Add dependency-boundary check. +- [x] Add backend permutation startup smoke checks. +- [x] Inventory access/auth/RBAC import debt. +- [x] Draft access extraction plan. +- [x] Define kernel access DTOs and protocols. +- [x] Add access capability registry names. +- [x] Register live access directory and tenant resolver capability + implementations backed by the current compatibility tables. +- [x] Register an access tenant-provisioner capability so tenancy can seed + access-owned default roles without importing access service internals. +- [x] Replace `govoplan-files` direct user/group/tenant model imports with the + `access.directory` capability for group membership and share-target checks. +- [x] Replace `govoplan-files` direct campaign model imports with the + `campaigns.access` capability for campaign share/existence checks. +- [x] Replace `govoplan-mail` direct user/group/tenant model imports with + mail-owned policy storage plus access-directory validation and legacy-read + fallback. +- [x] Replace `govoplan-mail` direct campaign model imports with the + `campaigns.mailPolicyContext` capability for campaign-scoped mail policy and + owner context. +- [x] Replace `govoplan-campaign` runtime user/group/tenant model imports with + access-directory lookups and string-based ORM relationships. +- [x] Create `govoplan-access` repository workspace and issue workflow scaffold. +- [x] Create `govoplan-access` repository/package skeleton. +- [x] Move `govoplan_core/access` seed package into `govoplan-access`. +- [x] Add compatibility wrappers in core for old import paths. +- [x] Route existing auth dependency wrappers through access principal/evaluator capabilities. +- [x] Move FastAPI auth dependency wrappers out of core and into + `govoplan-access`. +- [x] Move session, API-key, and password helper services into `govoplan-access`. +- [x] Move interactive auth/session routes behind access module manifest. +- [x] Move legacy admin/API-key routes behind access module manifest. +- [x] Move access-owned legacy admin service helpers into `govoplan-access`. +- [x] Move governance-template CRUD helpers and routes into `govoplan-admin`. +- [x] Move governance-template materialization of access-owned groups and roles + behind the `access.governanceMaterializer` capability. +- [x] Replace legacy access-to-files/campaign admin lookups with module + tenant-summary and group-delete veto providers. +- [x] Split legacy admin route contribution across `govoplan-admin`, + `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit`, and + `govoplan-access` route slices. +- [x] Move legacy admin route-handler ownership out of the access compatibility + router into access, admin, tenancy, policy, and audit module routers. +- [x] Move shared system-settings, tenant-governance, slug/error, and + tenant-count helpers out of access internals into core settings/tenancy + helpers used by the split platform route modules. +- [x] Move campaign schema routes into the campaign route contribution. +- [x] Move development mailbox routes into the mail route contribution. +- [x] Replace core Celery direct campaign/mail imports with the + `campaigns.deliveryTasks` capability. +- [x] Replace core retention direct campaign queries with + `campaigns.policyContext` and `campaigns.retention` capabilities. +- [x] Replace core `create_all` feature-model imports with module registry + metadata discovery. +- [x] Remove transitional boundary checker allowlist entries. +- [x] Move live legacy model definitions out of core and into + `govoplan-access` while preserving existing table names. +- [x] Split the transitional access-owned legacy model graph further into + tenancy, audit, admin, access, and core settings ownership. +- [x] Reverse the tenancy/access dependency direction so `govoplan-access` + depends on `govoplan-tenancy`, and the registry inserts tenancy before access. +- [x] Replace tenancy-to-access default role seeding with an access + tenant-provisioner capability. +- [x] Replace tenancy-to-access owner candidate and owner membership handling + with the access tenant-provisioner capability. +- [x] Replace admin overview counts and audit actor lookup/filtering with the + `access.administration` capability. +- [x] Replace feature-module direct user/group/tenant model imports. + - [x] `govoplan-files` + - [x] `govoplan-mail` + - [x] `govoplan-campaign` +- [x] Replace files/mail/campaign direct cross-module SQL lookups with + provider/capability contracts. +- [x] Move access admin WebUI pages to module route contributions. +- [x] Move generic admin-owned WebUI panels into `govoplan-admin` and register + them through the `admin.sections` UI capability. +- [x] Remove transitional boundary checker allowlist entries as each contract + lands. +- [x] Remove old core import shims for access models, access routes, admin + service helpers, and access-owned security services. +- [ ] Move campaign-scoped mail policy ownership fully behind an API/event + workflow if direct synchronous capability calls become too tight for later + deployment boundaries. + +## Open Decisions + +- Is `govoplan-access` required for any authenticated deployment, while + core-only remains a diagnostics/settings shell? +- Tenant models live in `govoplan-tenancy`; `govoplan-access` depends on + tenancy for authenticated platform composition. +- Historical table names remain stable for painless migrations. +- Should secret encryption remain a kernel primitive or become an access-owned + capability? +- Audit log storage lives in `govoplan-audit`; policy provenance can build on + that module boundary. +- How long should core keep compatibility route paths for existing clients? + +## Verification + +Use the following checks while extracting access: + +```bash +cd /mnt/DATA/git/govoplan-core +./.venv/bin/python scripts/check_dependency_boundaries.py +./.venv/bin/python -m unittest tests.test_module_system +``` + +For each removed dependency-boundary exception, add or update a focused test +that proves the replacement contract starts without the old direct import. diff --git a/docs/CATALOG_TRUST_AND_LICENSING.md b/docs/CATALOG_TRUST_AND_LICENSING.md new file mode 100644 index 0000000..f4422ea --- /dev/null +++ b/docs/CATALOG_TRUST_AND_LICENSING.md @@ -0,0 +1,185 @@ +# Module Catalog Trust And Licensing + +GovOPlaN module install and uninstall must remain operator-controlled. The +running server may plan and validate package changes, but package mutation is +performed by the separate installer daemon or an operator shell during +maintenance mode. + +## Roles + +`govoplan-web` is the public static distribution surface for official catalog +resources: + +- signed module package catalogs, grouped by release channel +- public catalog keyrings +- public license verification keyrings +- examples and operator-facing download paths + +`govoplan-core` is the verifier and orchestrator: + +- fetches a local or remote module catalog +- verifies catalog signatures against configured trusted keys +- enforces approved release channels +- rejects expired or not-yet-valid catalogs +- records accepted catalog sequence numbers for replay protection +- checks catalog entry license feature requirements before planning installs +- writes installer plans and request records + +Feature and platform modules own their package artifacts, manifests, migration +metadata, retirement providers, and optional lifecycle behavior. + +## Catalog Source + +Core accepts either a local catalog file or a remote URL: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG=/srv/govoplan/catalogs/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json +``` + +If both file and URL are set, the URL wins. The cache is used when a remote +fetch fails, so an operator can still inspect the last known catalog. A cached +catalog must still pass signature, freshness, channel, and replay validation. + +## Catalog Shape + +An official catalog is a JSON object with: + +- `catalog_version` +- `channel` +- `sequence` +- `generated_at` +- `not_before` when delayed activation is needed +- `expires_at` +- `modules` +- `signatures` + +Each module entry can declare: + +- backend package name and pinned install reference +- WebUI package name and pinned install reference +- display metadata and tags +- `license_features`, the feature entitlements required to plan that install + +The signature is Ed25519 over canonical JSON with both `signature` and +`signatures` removed. Core accepts the legacy single `signature` field and the +new `signatures` array. + +## Keyrings And Rotation + +Trusted catalog keys are configured locally: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":""}' +``` + +For development or tightly controlled deployments, a keyring can be read from a +URL and cached: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL=https://govoplan.example/catalogs/v1/keyring.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE=/srv/govoplan/runtime/catalog-cache/keyring.json +``` + +Production installations should pin the trusted keyring locally or ship it +through deployment configuration. Fetching trusted keys from the same public +origin as the catalog is convenient, but that origin must not become the only +trust root. + +Keyring entries support: + +- `key_id` +- `public_key` or `public_key_base64` +- `status`: `active`, `next`, `retired`, `revoked`, or `disabled` +- `not_before` +- `not_after` + +Rotation process: + +1. Add the next public key to the local trusted keyring with status `next`. +2. Publish catalogs signed by both current and next keys. +3. Upgrade installations so the next key is locally trusted. +4. Promote the next key to `active`. +5. Retire the old key only after every supported installation trusts the new + key. +6. Mark a compromised key `revoked` and publish a higher sequence catalog + signed by an uncompromised key. + +## Replay And Freshness + +Use replay state in production: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true +``` + +Core records the accepted sequence per channel after a catalog entry is planned +from the admin interface. With strict sequence enforcement, a previously +accepted sequence is rejected; without strict enforcement, only older sequences +are rejected. + +Catalogs should always expire. Long-lived catalogs make rollback and key +compromise harder to reason about. + +## Release Channels + +Approved channels are deployment policy: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable,lts +GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true +``` + +The admin UI can display other catalog metadata, but core rejects catalogs from +unapproved channels when validation is configured. + +## Licensing + +Catalog entries can require license features: + +```json +"license_features": ["module.mail", "support.standard"] +``` + +Core checks those requirements against an offline license file before allowing +the entry into the install plan. + +```bash +GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json +GOVOPLAN_LICENSE_ENFORCEMENT=true +GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json +``` + +License files are JSON objects with: + +- `license_id` +- `subject` +- `features` +- `valid_from` +- `valid_until` +- `signature` + +License enforcement can run in observe-only mode by leaving +`GOVOPLAN_LICENSE_ENFORCEMENT` unset. In that mode, missing or invalid license +data is surfaced as a warning but does not block planning. + +Licensing is intentionally separate from open-source code licensing. The +catalog/license mechanism can govern support channels, official release +eligibility, hosted update access, professional support, or commercial +entitlements without changing the source license of the repositories. + +## Production Gaps + +The current implementation validates signed catalog metadata and offline +license entitlements. Production-grade distribution still needs: + +- artifact digest verification for package archives or registry artifacts +- SBOM/provenance fields and verification workflow +- hardened catalog publishing pipeline in `govoplan-web` +- automated key rotation runbook and emergency revocation procedure +- license issuance and renewal tooling +- tests for remote catalog cache fallback and replay-state upgrades + diff --git a/docs/CODEX_WORKFLOW.md b/docs/CODEX_WORKFLOW.md index 4a09c76..46c7dc2 100644 --- a/docs/CODEX_WORKFLOW.md +++ b/docs/CODEX_WORKFLOW.md @@ -24,6 +24,9 @@ network_access = false [projects."/mnt/DATA/git/govoplan-core"] trust_level = "trusted" +[projects."/mnt/DATA/git/govoplan-access"] +trust_level = "trusted" + [projects."/mnt/DATA/git/govoplan-mail"] trust_level = "trusted" @@ -42,6 +45,8 @@ Each active repository has an `AGENTS.md` file. These files define ownership, mo Use `~/.codex/config.toml` for personal defaults, auth/runtime settings, writable roots, and trust decisions. Avoid checking in absolute-path writable roots or model preferences unless they are intentionally team-wide. +Use Gitea issues as the canonical backlog and state log. See `docs/GITEA_ISSUES.md` for label setup, TODO import, and Codex issue update commands. Durable docs should describe stable behavior; changing work state belongs on the issue. + ## Focused Verification Use the consolidated script after changes that touch module discovery, optional integrations, shared mail components, mailbox listing, or cross-module WebUI behavior: @@ -70,4 +75,5 @@ PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versi - Avoid broad recursive scans and full builds unless the change warrants them. - Keep generated build/test folders ignored. - Keep optional module behavior behind core registry/capability/module metadata boundaries. +- Create or update Gitea issues for TODOs, follow-ups, blockers, and feature requests instead of keeping local backlog files. - Do not start persistent dev servers unless the user asks. diff --git a/docs/CONFIGURATION_PACKAGES.md b/docs/CONFIGURATION_PACKAGES.md new file mode 100644 index 0000000..d2fb858 --- /dev/null +++ b/docs/CONFIGURATION_PACKAGES.md @@ -0,0 +1,287 @@ +# GovOPlaN Configuration Packages + +Configuration packages are reusable, versioned preconfigurations for a working +GovOPlaN capability. They sit above modules: a module installs code, +migrations, routes, permissions, and capabilities; a configuration package wires +installed modules into a concrete operational setup. + +Example: an application-handling package could configure a public portal form, +a case workflow, task creation, a mail template, payment processing, access +roles, audit evidence, and the interface bindings between those modules. + +This document is durable architecture context and should be mirrored to the +Gitea wiki. Active implementation should be tracked in Gitea issues. + +## Goals + +- Import a preconfiguration from a trusted catalog or uploaded package. +- Detect which modules, module versions, and capabilities are required. +- Detect which configuration fragments must be created, updated, or bound. +- Explain import problems and offer actionable resolution paths. +- Ask the operator only for data needed to make the package work. +- Export an existing working configuration as a reusable package. +- Support signed catalogs so approved configurations can be shared, adapted, + re-exported, and provided to other installations. + +## Vocabulary + +| Term | Meaning | +| --- | --- | +| Module | Installable product code with manifest, migrations, routes, permissions, WebUI, events, and capabilities. | +| Configuration | Module-owned settings and resources that make behavior concrete: workflows, forms, templates, roles, policies, connector profiles, routing rules, and defaults. | +| Interface | A typed contract between modules: capabilities, commands, events, DTOs, data bindings, and UI extension points. | +| Data | Operator- or tenant-provided values needed for a working setup: legal text, sender addresses, account mappings, payment provider credentials, templates, defaults, and optionally reference data. | + +The system should communicate these four layers explicitly: + +```text +module = what can exist +configuration = what should exist here +interface = how configured parts connect +data = what the operator must provide for this deployment +``` + +## Package Model + +A configuration package should be a signed, portable manifest plus module-owned +configuration fragments. The package is not a database dump. It should be +declarative, idempotent, tenant-aware, and explicit about secrets and external +systems. + +Required package metadata: + +- stable package id, name, version, description, publisher, and license +- supported GovOPlaN core/module compatibility bounds +- required modules with version constraints +- optional modules with conditional behavior +- required capabilities, commands, event subscriptions, and UI extension points +- configuration fragments grouped by owning module +- interface bindings between fragments +- data requirements to collect from the operator +- preflight checks and post-import health checks +- migration or transformation rules for older package versions +- provenance, export source metadata, and signature metadata + +Configuration fragments are interpreted only by the module that owns them. For +example, workflow imports workflow definitions; forms imports form schemas; +mail imports mail templates and delivery defaults; payments imports payment +profiles; access imports groups, roles, and permission assignments. + +## Module Provider Contract + +Each module that wants to participate should expose a configuration-package +provider capability. The core orchestrator should not understand module internals +or write module-owned tables directly. + +Baseline provider responsibilities: + +- publish JSON Schema or equivalent typed schemas for importable/exportable + fragments +- publish data requirements that can be rendered by a generic wizard +- validate fragments without applying them +- report missing dependencies, missing permissions, conflicts, and unsafe + changes +- produce a dry-run plan with create, update, bind, skip, and blocked actions +- apply fragments idempotently inside module-owned boundaries +- export selected module-owned configuration into portable fragments +- redact secrets and mark secret placeholders during export +- provide post-apply health checks and operator-facing diagnostics + +Provider operations should be versioned. The first stable contract can be small: + +```text +describe() -> schemas, supported fragment types, exported scopes +preflight(package_fragment, context) -> diagnostics, required_data, plan +apply(package_fragment, supplied_data, context) -> result, created_refs +export(selection, context) -> fragment, data_placeholders, warnings +health(import_result, context) -> diagnostics +``` + +## Import Flow + +1. Select a package from a trusted catalog or upload a package file. +2. Verify signature, channel, publisher trust, package compatibility, and schema. +3. Resolve required modules against installed modules and the module package + catalog. +4. Produce a dependency plan for modules that must be installed or upgraded. +5. Ask the operator for required data using a focused wizard. +6. Run dry-run preflight across all participating module providers. +7. Show problems grouped by severity, owner module, and resolution. +8. Apply module/package changes in a dependency-safe order. +9. Run post-import health checks and show the final working status. +10. Store import provenance, package version, supplied non-secret metadata, and + audit events. + +The wizard should display everything necessary and nothing unnecessary. Generic +sections should cover package trust, dependency plan, required data, conflicts, +review, and result. Module-specific fields should appear only when the selected +package and installed modules require them. + +## Problem Model + +Import diagnostics should be structured and actionable, not free-text logs. +Every problem should include severity, owner, affected object, explanation, and +at least one suggested resolution when the system can infer it. + +Common diagnostic types: + +- missing module or incompatible module version +- missing capability or disabled optional integration +- missing operator-supplied data +- missing permission or insufficient administrator scope +- unresolved interface binding between modules +- conflicting existing configuration +- external service not reachable or credential test failed +- policy or tenant-governance violation +- unsafe overwrite, downgrade, or destructive change +- schema validation failure +- post-import health-check failure + +Resolution actions can include install module, upgrade module, enable +integration, provide value, choose existing object, rename imported object, +skip optional fragment, replace existing configuration, or cancel import. + +## Export Flow + +Export should be possible from a working tenant or system configuration, but it +must be intentional about data boundaries. + +1. Select export scope: system, tenant, module set, workflow, form, case type, + portal flow, or another domain object. +2. Ask participating modules to export owned fragments and data placeholders. +3. Classify exported material as configuration, reference data, sample data, + secrets, or deployment-local data. +4. Redact secrets by default and replace them with data requirements. +5. Let the operator choose whether to include reference/sample data. +6. Validate the assembled package by running the same preflight path against a + clean target context where possible. +7. Sign the package or produce an unsigned development package. +8. Optionally publish the package metadata to a configuration catalog. + +Exported packages should record provenance: source GovOPlaN version, module +versions, exporter identity, timestamp, selected scope, redactions, and +validation status. + +## Catalogs And Trust + +Configuration catalogs should follow the existing module package catalog model: +a file-backed or remotely fetched JSON catalog with Ed25519 signatures, channel +gating, trusted key ids, and operator-controlled trust policy. + +Catalog entries should include: + +- package id, version, name, description, publisher, tags, and channel +- package artifact URL or repository ref +- signature metadata for the package artifact +- required modules and version ranges for quick compatibility display +- package category such as workflow, portal, case type, governance, connector, + report, or tenant baseline +- maturity flags such as official, verified, example, local, deprecated + +The catalog verifies that a package is approved to view and import. The package +itself must still be signed and validated before use. + +## Example Package + +Application handling through a public portal: + +Required modules: + +- `portal` for the public user-facing entry point +- `forms` for the application form and validation +- `cases` for the case record and case type +- `workflow` for status transitions and automation +- `tasks` for internal task creation +- `templates` for mail/template rendering +- `mail` for delivery profile and outbound message sending +- `payments` for provider configuration and payment capture +- `access` for roles, groups, and permissions +- `audit` for import, case, mail, and payment evidence + +Required configuration: + +- portal route and access policy +- form schema, validation rules, confirmation texts, and attachments +- case type, fields, lifecycle states, and retention classification +- workflow steps, transitions, assignees, and completion triggers +- task template and queue/group assignment +- mail template and delivery rule +- payment product, amount rules, provider profile, and webhook binding +- access roles/groups for clerks, reviewers, supervisors, and operators +- audit event categories and evidence retention defaults + +Required operator data: + +- tenant or organizational unit +- service name and public contact details +- portal URL/path and legal imprint/privacy text +- mail sender profile and reply-to mailbox +- payment provider credentials or test-mode selection +- responsible groups or users for task assignment +- escalation contacts and deadlines +- template wording and localized text overrides + +Potential problems: + +- `payments` is missing or the provider capability is unavailable +- mail transport test fails for the selected sender profile +- imported role names conflict with existing tenant roles +- workflow references a task queue that does not exist +- public portal base URL is not configured +- required privacy/legal text is empty +- webhook endpoint cannot be validated from the payment provider + +## Ownership + +Core should own: + +- package and catalog signature validation +- dependency resolution against installed modules and module catalogs +- orchestration of provider preflight/apply/export operations +- generic import/export APIs and audit envelope +- generic wizard shell and problem display components + +Modules should own: + +- schemas and semantics for their own fragments +- module-specific validation, apply, export, and health checks +- module-specific UI capabilities only when generic generated controls are not + sufficient +- redaction and classification of module-owned secrets or sensitive data + +Access should own configuration fragments for users, groups, roles, +permissions, API keys, and principal mappings. It should not own workflow, mail, +payment, form, or portal semantics. + +Admin should expose the operator-facing catalog, import, export, and history +screens through admin route contributions. The UI should keep the conceptual +layers visible: modules, configuration, interfaces, and data. + +## Implementation Slices + +1. Define package manifest and diagnostic schemas. +2. Add core configuration-package provider capability contracts. +3. Implement catalog validation and package signature verification. +4. Add dry-run orchestration against mock providers. +5. Add admin catalog/import wizard screens using provider data requirements. +6. Implement export/import providers for access-owned roles/groups first. +7. Add providers in workflow/forms/templates/mail/payments/portal/cases/tasks as + those modules mature. +8. Add round-trip tests: export a known setup, import into a clean tenant, + verify health checks. +9. Add documentation and field-level help for package authors and operators. + +## Acceptance Criteria For Tracking Issue + +- A Gitea tracking issue exists in `govoplan-core` for the cross-cutting + feature. +- The issue links module-specific follow-ups when implementation begins. +- The first implementation exposes typed contracts rather than direct + cross-module imports. +- A signed example catalog and unsigned development fixture exist. +- A dry-run import can identify required modules, missing data, and conflicts + before applying changes. +- An export can produce a package with secrets redacted into data requirements. +- Admin UI shows a focused wizard and actionable diagnostic list. +- Tests cover package validation, signature failure, dependency resolution, + provider preflight, export redaction, and import idempotency. diff --git a/docs/GITEA_ISSUES.md b/docs/GITEA_ISSUES.md new file mode 100644 index 0000000..382a3e4 --- /dev/null +++ b/docs/GITEA_ISSUES.md @@ -0,0 +1,288 @@ +# Gitea Issues And Wiki Workflow + +Gitea issues are the canonical backlog for GovOPlaN work: bugs, feature requests, tasks, tech debt, TODO migrations, open decisions, and blocked work should live there. Gitea wiki pages are the canonical project reference for durable project context mirrored from repository docs and product-directory notes. + +The same pattern is reusable outside GovOPlaN for any project where Codex works in a local checkout, VSCodium or another editor is used for human inspection, and Gitea is the issue tracker. In that setup, Gitea is the durable coordination layer; Codex and the editor are clients of that state. + +## Initial Setup + +The repository contains Gitea issue templates in `.gitea/ISSUE_TEMPLATE`, a pull request template in `.gitea/PULL_REQUEST_TEMPLATE.md`, and the label taxonomy in `docs/gitea-labels.json`. + +The scripts infer this repository from `origin` (`git@git.add-ideas.de:add-ideas/govoplan-core.git`). Override inference when needed: + +```bash +export GITEA_URL=https://git.add-ideas.de +export GITEA_OWNER=add-ideas +export GITEA_REPO=govoplan-core +export GITEA_TOKEN=... +``` + +The API scripts also read `GITEA_*` values from the target repository's `.env` file. That file is gitignored in this repo, so it is suitable for local tokens: + +```bash +GITEA_TOKEN=... +# Optional if origin inference is not enough: +GITEA_URL=https://git.add-ideas.de +GITEA_OWNER=add-ideas +GITEA_REPO=govoplan-core +``` + +For a shared credentials file outside the target repository, pass `--env-file`: + +```bash +./scripts/gitea-sync-labels.py --env-file /path/to/private/gitea.env --apply +``` + +Create a Gitea token with issue read/write access and label-management permission for the repository. On scoped-token instances, this usually means issue read/write and, if label writes are rejected, repository write permission too. + +Preview and apply labels: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/gitea-sync-labels.py +./scripts/gitea-sync-labels.py --apply +``` + +After the `.gitea` files are pushed to the default branch, Gitea will show the issue template chooser. Blank issues are disabled by `.gitea/ISSUE_TEMPLATE/config.yaml`. + +## Multiple Repositories And Workspaces + +The helper scripts are path-based and can run from this core checkout against any repository with a Gitea remote: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/gitea-sync-labels.py --root /mnt/DATA/git/govoplan-mail --apply +./scripts/gitea-todo-import.py --root /mnt/DATA/git/govoplan-mail +./scripts/gitea-codex-note.py --root /mnt/DATA/git/govoplan-mail --issue 123 --status progress +``` + +Each target repository is inferred from its own `origin` remote. Use `GITEA_URL`, `GITEA_OWNER`, or `GITEA_REPO` only when a workspace has unusual remotes or the Gitea web URL cannot be inferred from SSH. + +When one core checkout drives another workspace, prefer `--env-file` for shared credentials instead of putting `GITEA_REPO` in the core `.env`; a repo-specific `GITEA_REPO` can accidentally override target inference. + +For non-GovOPlaN projects, provide a project-specific label file and module/project label: + +```bash +./scripts/gitea-sync-labels.py \ + --root /path/to/project \ + --labels-file /path/to/project/docs/gitea-labels.json \ + --apply + +./scripts/gitea-todo-import.py \ + --root /path/to/project \ + --module-label project/example \ + --extra-label area/backend +``` + +If another project does not use `area/*` labels, disable area inference: + +```bash +./scripts/gitea-todo-import.py \ + --root /path/to/project \ + --module-label project/example \ + --no-area-labels +``` + +Install or refresh the shared issue templates in sibling or external repositories: + +```bash +./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail +./scripts/gitea-install-workflow.py /mnt/DATA/git/govoplan-mail --apply +``` + +The installer rewrites the default template label from `module/core` to the module label inferred from the target repository name. Known mappings cover the packaged GovOPlaN repositories, and any other `govoplan-` checkout maps to `module/`. For another workspace or repository name, pass an explicit label: + +```bash +./scripts/gitea-install-workflow.py /path/to/repo --module-label module/example --apply +``` + +Use `--include-labels-file` if a repository should carry its own copy of `docs/gitea-labels.json`; otherwise keep the shared taxonomy in core and run the sync script from core. + +For a fully portable workflow kit, copy these files into the other project: + +- `scripts/gitea_common.py` +- `scripts/gitea-sync-labels.py` +- `scripts/gitea-todo-import.py` +- `scripts/gitea-codex-note.py` +- `scripts/gitea-install-workflow.py` +- `.gitea/ISSUE_TEMPLATE/*` +- `.gitea/PULL_REQUEST_TEMPLATE.md` +- a project-specific `docs/gitea-labels.json` + +Keep credentials out of the repository. Put `GITEA_TOKEN` in the shell environment, a gitignored `.env`, a local direnv file, or the user-level Codex/VSCodium environment setup. + +## Label Taxonomy + +Use one `type/*` label: + +- `type/bug` +- `type/feature` +- `type/task` +- `type/debt` +- `type/docs` + +Use one `status/*` label while the issue is open: + +- `status/triage`: needs ownership, priority, or acceptance criteria. +- `status/ready`: ready to implement. +- `status/in-progress`: actively being worked. +- `status/blocked`: blocked on an external dependency, credential, or decision. +- `status/needs-info`: blocked on clarification. + +Use one `priority/*` label when prioritization matters: `priority/p0`, `priority/p1`, `priority/p2`, or `priority/p3`. + +Use `module/*` and `area/*` labels to route work. Module labels are not exclusive because cross-module work can exist. Core issues should still preserve ownership boundaries: module-specific implementation belongs in the owning module repository. + +Use `codex/ready` when the issue has enough context for Codex to work from, and `codex/needs-human` when a human decision is required first. + +## Moving TODOs Into Gitea + +Preview inline markers: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/gitea-todo-import.py +``` + +Create missing issues after labels are synced: + +```bash +./scripts/gitea-todo-import.py --apply +``` + +The importer scans `TODO`, `FIXME`, `XXX`, and `HACK` markers, skips markers that already reference an issue, applies `source/todo-scan`, and writes a hidden fingerprint into each generated issue body so reruns do not duplicate already imported items. + +When touching code with an imported marker, either remove the marker as part of the fix or replace it with a short reference: + +```python +# TODO(gitea#123): keep only if the local pointer is still useful +``` + +Do not add new untracked TODO comments. Create the Gitea issue first, then reference it inline only when the local pointer materially helps future readers. + +For a broader project import across all local repositories hosted on `git.add-ideas.de`, use the generic backlog importer: + +```bash +./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env +./scripts/gitea-import-all-backlogs.py --env-file /home/zemion/.config/gitea/gitea.env --apply +``` + +It scans repository and product-directory files with backlog-like names, creates the shared generic labels where needed, imports missing open work, and deduplicates reruns by hidden fingerprint and normalized title. + +## Mirroring Project Docs Into Gitea Wikis + +Preview wiki pages for all local repositories hosted on `git.add-ideas.de`, cross-referenced with product directories under `/mnt/DATA/Nextcloud/ADD ideas UG/Products`: + +```bash +cd /mnt/DATA/git/govoplan-core +./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env +``` + +Apply the wiki mirror: + +```bash +./scripts/gitea-sync-wiki.py --env-file /home/zemion/.config/gitea/gitea.env --apply +``` + +The default apply path uses the Gitea wiki git repository, not one REST API +request per page. It keeps a local checkout cache below +`/tmp/codex-gitea-wiki-sync`, commits changed pages once per repository, and +pushes that commit. This is much faster and avoids REST wiki-page timeouts. +Use `--transport api` only when the wiki git remote is unavailable. + +Limit a sync to one repository or one generated page while working: + +```bash +./scripts/gitea-sync-wiki.py --repo govoplan-core --apply +./scripts/gitea-sync-wiki.py --repo govoplan-core --page Repo-docs-MODULE-ARCHITECTURE --apply +``` + +Page-limited syncs do not rewrite `Codex-Project-Index`; run a full repository +sync when the set of mirrored pages changes. + +The wiki sync mirrors durable text documents only: root README-style files, docs/codex project docs, and selected product notes such as roadmap, plan, concept, pitch, and whitepaper files. It skips generated folders, dependency/build output, `.gitea` templates, and filenames that look credential-related. + +Each managed wiki page contains a `codex-wiki-sync` marker and a source path. Reruns update only managed pages unless `--overwrite-unmanaged` is passed. Each repository also gets a managed `Codex-Project-Index` page linking the mirrored pages. + +Use the wiki for durable context: + +- project overviews and architecture +- workflows, operating notes, and setup references +- product concepts, plans, pitches, and whitepapers +- historical context that helps interpret issues + +Keep active state in issues: + +- open tasks, TODOs, feature requests, and bugs +- priority, blocking status, and acceptance criteria +- Codex progress updates and implementation notes + +## Codex State Updates + +Codex should read the relevant issue before making changes when issue access is available. During or after work, Codex should add issue comments with the state that would otherwise drift into local notes: + +- scope understood +- files changed +- tests or manual checks run +- blockers or decisions needed +- follow-up issues created + +Preview and post a standardized note: + +```bash +./scripts/gitea-codex-note.py \ + --issue 123 \ + --status progress \ + --summary "Implemented capability metadata fallback." \ + --changed src/govoplan_core/modules/registry.py \ + --test "./.venv/bin/python -m unittest tests.test_module_system" + +./scripts/gitea-codex-note.py \ + --issue 123 \ + --status progress \ + --summary "Implemented capability metadata fallback." \ + --changed src/govoplan_core/modules/registry.py \ + --test "./.venv/bin/python -m unittest tests.test_module_system" \ + --apply +``` + +Use `--close --apply` only when the acceptance criteria are satisfied and verification is recorded. + +## Ownership Rules + +Create the issue in the repository that owns the change: + +- `govoplan-core`: platform runner, DB/session primitives, auth, tenancy, RBAC, governance, module discovery, migrations, shared WebUI shell, and generic WebUI components. +- `govoplan-access`: access, identity, authentication, sessions, API keys, RBAC, groups, users, and access administration. +- `govoplan-mail`: mail-specific backend, frontend, message workflows, and mail integrations. +- `govoplan-files`: files-specific backend, frontend, storage, and file workflows. +- `govoplan-campaign`: campaign-specific backend, frontend, policy, and template behavior. + +For cross-cutting work, create a tracking issue in `govoplan-core` and link module issues from it. Do not use the core issue as a dumping ground for module-specific implementation details. + +## Cleaning Up Mirrored Sources + +After backlog files have been imported into issues and durable context has been mirrored to wiki, old duplicate sources can be removed from git only when they are tracked files and the Gitea issue/wiki state has been verified. Prefer deleting backlog, TODO, roadmap, and one-off planning files that have become duplicate state. + +Do not delete standard repository entry points such as `README`, `LICENSE`, `SECURITY`, or package metadata just because they are mirrored to the wiki. They remain useful for repository browsing, package registries, and developer onboarding. + +Do not delete untracked files or files outside git history as part of automated cleanup unless there is a separate backup or explicit human confirmation for that specific path. + +## Docs Versus Issues + +Keep durable facts in docs: + +- architecture and extension points +- command references +- module boundaries +- operational conventions + +Keep changing state in Gitea: + +- TODOs and follow-ups +- bugs and feature requests +- blocked status +- acceptance criteria +- implementation notes from active work + +If a decision becomes durable architecture, write the durable result into docs and link back to the issue for history. diff --git a/docs/GOVOPLAN_MODULE_ROADMAP.md b/docs/GOVOPLAN_MODULE_ROADMAP.md new file mode 100644 index 0000000..26eb760 --- /dev/null +++ b/docs/GOVOPLAN_MODULE_ROADMAP.md @@ -0,0 +1,79 @@ +# GovOPlaN Module And Integration Roadmap + +This page maps current module and integration ideas to existing GovOPlaN repositories or to missing-module decisions. Issues are the active backlog. This document is durable routing context and should be mirrored to the Gitea wiki. + +## Current Routing + +| Idea | Owner | Tracking | +| --- | --- | --- | +| Access as a module | `govoplan-access` | `add-ideas/govoplan-access#7` | +| OpenProject API / project management connector | `govoplan-connectors` | `add-ideas/govoplan-connectors#1` | +| Native project-management module decision | `govoplan-core` | `add-ideas/govoplan-core#196` | +| Datasources for databases, CSV, files, APIs | proposed `govoplan-datasources` | `add-ideas/govoplan-core#197` | +| Dataflow for pipelines, BI, publication | proposed `govoplan-dataflow` | `add-ideas/govoplan-core#198` | +| Templates for letters, emails, forms, reports | `govoplan-templates` | `add-ideas/govoplan-templates#1` | +| Reporting and BI | `govoplan-reporting` | `add-ideas/govoplan-reporting#1` | +| File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `add-ideas/govoplan-files#15` | +| Public-sector software integration catalogue | `govoplan-connectors` | `add-ideas/govoplan-connectors#2` | +| Adrema-style address and distribution-list management | `govoplan-addresses` | `add-ideas/govoplan-addresses#1` | +| Consume sources and become a governed source | `govoplan-connectors` plus proposed `govoplan-dataflow` | `add-ideas/govoplan-connectors#3`, `add-ideas/govoplan-core#198` | +| Terminfindung and meeting scheduling polls | `govoplan-scheduling` | local scaffold; remote creation tracked by `add-ideas/govoplan-core#199` | +| Terminplaner and calendar primitives | `govoplan-calendar` | `add-ideas/govoplan-calendar#1` | +| Terminbuchung appointment booking | `govoplan-appointments` | `add-ideas/govoplan-appointments#1` | +| Collaborative documents | `govoplan-dms` | `add-ideas/govoplan-dms#1` | +| Forms | `govoplan-forms` | `add-ideas/govoplan-forms#1` | +| RSS consume and emit | `govoplan-connectors` | `add-ideas/govoplan-connectors#4` | +| LDAP, Active Directory, OpenDesk identity | `govoplan-idm` | `add-ideas/govoplan-idm#1` | +| OpenDesk stack integration map | `govoplan-connectors` | `add-ideas/govoplan-connectors#5` | +| Open-Xchange mail/groupware | `govoplan-mail` | `add-ideas/govoplan-mail#5` | +| Open-Xchange calendar | `govoplan-calendar` | `add-ideas/govoplan-calendar#2` | + +## Proposed Missing Modules + +`govoplan-datasources` should exist only if source catalog ownership becomes broad enough to justify a module separate from connectors and reporting. Candidate responsibilities: + +- source catalogue for SQL databases, CSV/Excel files, uploaded files, APIs, RSS feeds, and governed file locations +- credentials and connection profiles +- schema discovery and refresh cadence +- provenance, freshness, permission boundaries, and audit events + +`govoplan-dataflow` should exist only if pipelines and publication become first-class product behavior. Candidate responsibilities: + +- ingestion, transformation, validation, scheduling, and lineage +- connecting datasources to reports, APIs, RSS, exports, and downstream systems +- the "consume sources, become source" lifecycle +- publication-state and audit integration + +`govoplan-projects` is undecided. The default path should be an OpenProject connector first. A native module is justified only if GovOPlaN must own project semantics beyond cases, tasks, workflow, appointments, documents, and reporting. + +## Boundary Notes + +- `govoplan-connectors` owns protocol and external-system integration strategy. It should not own business semantics once a domain module exists. +- `govoplan-files` owns file storage semantics and file-provider contracts. Remote provider implementations must stay optional. +- `govoplan-dms` owns document lifecycle, collaboration, versions, approvals, locks, retention, and legal hold. +- `govoplan-addresses` owns persons, organizations, postal/email addresses, distribution lists, and recipient import/export. +- `govoplan-calendar` owns events, availability, resources, recurrence, and groupware calendar adapters. +- `govoplan-scheduling` owns meeting scheduling polls, participant availability collection, candidate-slot ranking, and decision handoff. +- `govoplan-appointments` owns public/internal appointment-booking workflows. +- `govoplan-idm` owns directory, provisioning, and identity-provider integration; `govoplan-access` consumes resolved principals and permissions. +- `govoplan-reporting` owns report definitions, BI views, scheduled outputs, and export targets. +- `govoplan-templates` owns reusable renderable templates, not data selection or persistence. + +## Release Tooling Note + +`scripts/push-release-tag.sh` covers the released full-product package set: + +- `govoplan-access` +- `govoplan-admin` +- `govoplan-tenancy` +- `govoplan-policy` +- `govoplan-audit` +- `govoplan-files` +- `govoplan-mail` +- `govoplan-campaign` +- `govoplan-calendar` +- `govoplan-core` + +It also includes existing roadmap/scaffold module repositories such as addresses, appointments, connectors, DMS, forms, IDM, reporting, scheduling, templates, workflow, XOE/V, and XRechnung as tag-only repositories. Tag-only repositories are committed, tagged, and pushed with the same release tag, but they are not added to `requirements-release.txt` or `webui/package.release.json` until they contain installable package metadata. + +Proposed modules without repositories, such as `govoplan-datasources`, `govoplan-dataflow`, and possibly `govoplan-projects`, cannot be included until their repositories are created. diff --git a/docs/MODULE_ARCHITECTURE.md b/docs/MODULE_ARCHITECTURE.md index 07cd9d7..1598fa3 100644 --- a/docs/MODULE_ARCHITECTURE.md +++ b/docs/MODULE_ARCHITECTURE.md @@ -1,19 +1,147 @@ # GovOPlaN Module Architecture -GovOPlaN is structured as a core platform runner plus installable feature modules. Core owns the runtime shell and cross-cutting primitives. Modules own feature behavior and contribute backend routes, database metadata, permissions, WebUI routes, and navigation metadata. +GovOPlaN is structured as a platform kernel plus installable modules. The kernel starts and composes the platform. Modules own product behavior and contribute backend routes, database metadata, permissions, WebUI routes, navigation metadata, capabilities, and events. -## Core Responsibilities +The current package name is still `govoplan-core`, but the architecture target is a smaller kernel. Access, tenancy, policy, audit, and admin semantics are platform-module responsibilities and should be extracted in stages. -Core owns: +The concrete access/auth/RBAC extraction path is tracked in +[`ACCESS_EXTRACTION_PLAN.md`](ACCESS_EXTRACTION_PLAN.md). + +## Layer Model + +| Layer | Purpose | Examples | +| --- | --- | --- | +| Kernel | Bootstraps and composes the platform | app factory, module registry, route aggregation, migration orchestration, capability/event contracts, health metadata | +| Platform modules | Cross-cutting governance capabilities | access, tenancy, policy, audit, admin, ops | +| Service modules | Reusable operational capabilities | files, mail, templates, recipients, notifications | +| Business modules | Public-sector workflows | campaigns, cases, forms, approvals, appointments | +| Connector modules | External system integration | FIT-Connect, XÖV/XTA, DMS/eAkte, ERP, IDM | + +## Kernel Responsibilities + +The kernel target owns: - the server entry point and platform configuration -- module discovery, registry validation, route aggregation, and platform metadata APIs -- database engine/session primitives and migration orchestration -- auth, tenants, RBAC, governance, audit, CSRF/API helpers, and secret helpers -- shared WebUI shell components such as `AppShell`, `IconRail`, `DataGrid`, `ExplorerTree`, `MessageDisplayPanel`, dialogs, loading frames, access boundaries, and form primitives +- module discovery, manifest validation, registry validation, route aggregation, and platform metadata APIs +- database engine/session lifecycle and module migration orchestration +- module install-plan validation, signed catalog verification, license entitlement checks, and installer request orchestration +- capability registry, command/event contracts, and lifecycle hooks +- shared WebUI shell contracts, generic WebUI components, and module route/nav rendering - centralized mapping from serializable icon names to renderable frontend icons +- health, OpenAPI aggregation, and runtime diagnostics -Core must not import module feature pages or module business logic directly. It should interact with modules through manifests, entry points, metadata, and route contributions. +The kernel must not own product semantics such as users, tenants, RBAC decisions, governance policies, audit storage, mail behavior, file behavior, or campaign behavior. Those belong to platform, service, or business modules. + +## Current Compatibility Responsibilities + +During the staged split, `govoplan-core` still contains compatibility surfaces +for access, auth, tenancy, RBAC, governance, audit, CSRF/API helpers, and +secret helpers. The extracted access implementation now lives in +`govoplan-access`; live legacy ORM table definitions have been split across +their platform owners while retaining historical table names. The old core +model, route, admin-service, and access-security import shims have been +removed; callers must use module-owned imports or kernel capabilities. The +remaining compatibility surfaces are temporary until the matching platform +modules are fully self-contained: + +- `govoplan-access` +- `govoplan-tenancy` +- `govoplan-policy` +- `govoplan-audit` +- `govoplan-admin` + +New code should avoid deepening these compatibility dependencies. Prefer explicit kernel contracts and module capabilities over direct imports. + +Core must not import module feature pages or module business logic directly. It should interact with modules through manifests, entry points, metadata, capabilities, events, and route contributions. + +## Stable Kernel Contracts + +The following contracts are the baseline API that modules can rely on: + +- `ModuleManifest` +- `ModuleCompatibility` +- module uninstall guard provider contract +- `MigrationSpec` +- route factory contract +- capability factory contract +- access DTO/protocol contracts in `govoplan_core.core.access` +- resource ACL provider contract +- tenant summary provider contract +- tenant delete-veto provider contract +- WebUI module contribution contract +- navigation metadata contract +- command/event envelope contract + +Changes to these contracts must be versioned or accompanied by compatibility shims. + +Known access-related capability names are defined in +`govoplan_core.core.access`, including: + +- `access.principalResolver` +- `access.directory` +- `access.permissionEvaluator` +- `access.resourceAccess` +- `access.tenantProvisioner` +- `access.administration` +- `access.governanceMaterializer` +- `tenancy.tenantResolver` +- `security.secretProvider` +- `audit.sink` + +`govoplan-access` currently registers `access.principalResolver`, +`access.permissionEvaluator`, `access.directory`, `access.tenantProvisioner`, +`access.administration`, and `access.governanceMaterializer`. +`govoplan-tenancy` registers `tenancy.tenantResolver`. The minimal +authenticated platform set is now `tenancy` plus `access`; the registry +inserts `tenancy` before `access` when only feature modules are requested. +Feature modules should prefer these capabilities over direct reads of +access/tenant ORM models when they need labels, group membership, default +access provisioning, counts, audit actor labels, or tenant metadata. + +FastAPI route dependencies for authenticated endpoints are access-owned and +published from `govoplan_access.backend.auth.dependencies`. Routers may import +that dependency module directly until a more generic request-principal adapter +exists; they must not import access ORM models or other access implementation +internals. + +Current live table ownership: + +- `govoplan-tenancy`: `tenants` +- `govoplan-access`: `accounts`, `users`, `groups`, `roles`, + `system_role_assignments`, `user_group_memberships`, + `user_role_assignments`, `group_role_assignments`, `api_keys`, + `auth_sessions` +- `govoplan-admin`: `governance_templates`, + `governance_template_assignments` +- `govoplan-audit`: `audit_log` +- `govoplan-core`: `system_settings` + +Current admin route ownership follows the same boundary: access contributes +users, groups, roles, system accounts/roles, auth, sessions, and API-key +administration; tenancy contributes tenant registry/settings routes; admin +contributes system settings, overview, and governance-template routes; audit +contributes audit-log routes. Governance template metadata and assignment +routes live in `govoplan-admin`; materializing those templates into +access-owned groups and roles is performed by the +`access.governanceMaterializer` capability. + +Current admin WebUI ownership mirrors that route split. `govoplan-access` +contributes the `/admin` route shell and admin nav item. Other platform modules +contribute individual admin sections through the `admin.sections` UI capability. +`govoplan-admin` contributes the overview, system settings, and governance +template sections through that capability. Access-owned tenant/user/group/role +sections remain in the access package until their owning platform modules take +them over. + +Cross-module feature contracts live under focused kernel contract modules. For +example, `govoplan_core.core.campaigns` defines +`campaigns.access`, `campaigns.mailPolicyContext`, +`campaigns.policyContext`, `campaigns.deliveryTasks`, and +`campaigns.retention`. The campaign module registers these capabilities so mail +can resolve campaign owner/policy context and delivery tasks, files can validate +campaign file-share access, and core retention can call campaign-owned cleanup +logic without importing campaign ORM models. Keep these contracts small +DTO/protocol surfaces and register concrete behavior from the owning module. ## Module Responsibilities @@ -26,6 +154,7 @@ A module owns: - module migrations and migration metadata - module permissions and role templates - module-specific schemas, policies, and domain rules +- module package metadata and retirement providers used by install/uninstall preflight - WebUI pages, feature-specific components, API clients, route contributions, and navigation metadata A module should not own generic platform UI. If a component is useful outside one module, move it to `@govoplan/core-webui` and parameterize it there before reusing it. @@ -44,12 +173,15 @@ files = "govoplan_files.backend.manifest:get_manifest" The manifest should declare: - `id`, `name`, `version` +- `compatibility` when the module needs a minimum/maximum core version or a + newer manifest contract - required `dependencies` and `optional_dependencies` - permissions and role templates - router factory - migration metadata and script location - frontend package metadata - navigation metadata using serializable icon names +- uninstall guard providers for data, migration, worker, or scheduler vetoes Backend nav metadata must use icon-name strings, not frontend components: @@ -74,6 +206,37 @@ Rules: - Keep module-owned tables and migrations in the module repository. - Keep cross-module foreign-key assumptions explicit and conservative. - Register module metadata in `MigrationSpec` so core can discover it. +- Optional module migrations may create multiple Alembic heads. Verification + should compare the database heads to the configured script heads instead of + assuming one linear revision when multiple modules are enabled. + +## Install, Uninstall, And Catalogs + +Core owns the install plan, signed catalog validation, license entitlement +check, maintenance-mode guard, replay state, and installer request queue. +Package mutation is performed by `govoplan-module-installer` outside the +FastAPI request process. + +Official catalogs can be served as static JSON from `govoplan-web`, but core +does not trust the website by location alone. A catalog must pass the configured +signature, channel, freshness, and replay rules before a catalog entry can be +planned. Catalog entries may declare `license_features`; core checks those +against the configured offline license before adding the entry to the install +plan. + +Modules should provide: + +- pinned backend and WebUI package refs for official catalog entries +- compatibility metadata in the module manifest +- lifecycle hooks when a runtime enable/disable action needs module-specific + work +- uninstall guards for persistent data, active workers, schedulers, or external + bindings +- retirement providers when destructive uninstall can safely drop or retire + module-owned data + +Uninstall remains non-destructive unless the operator explicitly requests +`destroy_data` and the module provides a retirement provider that supports it. ## WebUI Contract @@ -103,6 +266,30 @@ WebUI modules receive only the core route context: A module should call its own API client and module-owned backend routes. Shared API helpers should live in core only when they are truly platform-level concerns. +Modules can also contribute named UI capabilities for explicit extension +points. Capability values must be narrow, typed contracts, not imports from a +sibling feature package. For admin pages, modules contribute: + +```ts +const adminSections: AdminSectionsUiCapability = { + sections: [ + { + id: "system-settings", + label: "General", + group: "SYSTEM", + order: 10, + allOf: ["system:settings:read"], + render: ({ settings, auth }) => createElement(SystemSettingsPanel, { settings, auth }) + } + ] +}; +``` + +The access admin route shell collects all installed `admin.sections` +capabilities with `usePlatformUiCapabilities("admin.sections")`, filters them +by `anyOf`/`allOf`, and renders them without importing the contributing module's +components directly. + ## Icon Rules Icons are resolved centrally by core. @@ -112,6 +299,7 @@ Modules must provide icon names with `iconName` in frontend nav contributions an Current core icon names include: - `activity` +- `admin` - `campaign` - `dashboard` - `file` @@ -124,6 +312,9 @@ Current core icon names include: If a module needs a new navigation icon, add the name-to-component mapping in core first, then use the name in backend and frontend metadata. +The access module uses the `admin` icon for its `/admin` route. Core only +resolves that icon name; it does not hard-code the admin route in the rail. + ## Shared Component Rules Use this rule of thumb: @@ -137,6 +328,9 @@ Examples: - `ExplorerTree` is core because files, mailboxes, and future modules can all render hierarchical navigation. - `MessageDisplayPanel` is core because mail, campaign sending, and later audit/review surfaces can display message-like content. +- `AdminPageLayout`, `AdminIconButton`, and `AdminSelectionList` are core + because access, admin, tenancy, policy, and audit panels share the same admin + shell language. - `MailProfileManagement` remains in the mail module because it is specific to mail transport policies and profiles. ## Cross-Module Integration @@ -147,16 +341,298 @@ Rules: - Use core module metadata to check whether another module is installed. - Use backend APIs/events/service contracts for runtime cooperation. +- If a sibling module needs owner-specific data, expose a narrow DTO/protocol + capability from the owning module instead of importing its ORM models. - Keep UI integration declarative where possible: nav items, route contributions, context actions, and explicit extension points. - Avoid direct imports from one feature module into another feature module unless the imported package is a published API contract designed for that purpose. UI components should be promoted to core instead. +### Dependency Boundary Enforcement + +The repository includes `scripts/check_dependency_boundaries.py`. It enforces the current baseline: + +- kernel/core source may not add new direct imports of files/mail/campaign internals +- access source may not import files/mail/campaign internals +- feature modules may not import access implementation internals +- feature modules may not add new direct imports of sibling feature modules +- FastAPI routers may import the published + `govoplan_access.backend.auth.dependencies` dependency API +- the transitional allowlist is expected to stay empty + +Any future exception is extraction debt and must be temporary, documented in the +script with a reason, and removed when a capability/API/event contract replaces +it. + +## Module Lifecycle + +Core exposes the installed module catalog through the admin API and WebUI. The +current lifecycle model separates four states: + +- installed: the Python/WebUI package is available to the process +- active: the module is present in the running platform registry +- desired: the module should be active on the next server startup +- planned package change: an operator-reviewed package install/uninstall item + saved in system settings but not executed by the running server + +The admin module manager can change the desired enabled set and apply it to the +running server. It always keeps `tenancy`, `access`, and `admin` enabled when +saving through the admin UI, and it adds required module dependencies before +saving the desired state. On startup, core always keeps the minimum +authenticated platform set `tenancy`/`access` enabled and keeps `admin` enabled +when the operator configuration includes it. Unknown saved module ids are +ignored when the matching package is no longer installed. The core app factory, +devserver, development bootstrap, background worker registry, and migration +metadata plan all read the saved desired state from `system_settings` before +building their module registry. + +Hot enable/disable is a core design principle for every module: + +- Core keeps one mutable active `PlatformRegistry` object and swaps its manifest + set through the module lifecycle manager. Modules must read module presence, + optional integrations, permissions, role templates, capabilities, navigation, + and frontend contributions from that registry instead of caching sibling + module availability. +- Core validates install state and dependency closure before activation. +- Core applies configured module migrations before activation. Deactivation + never drops tables or data. +- Core mounts module routers once and guards them by active module state. A + deactivated module's routes remain mounted internally but return a disabled + module response until the module is active again. +- Module route factories must be side-effect-light and idempotent. They may + configure module runtime references, but they must not start workers, + schedulers, or irreversible external subscriptions. Use lifecycle hooks for + those resources. +- Modules that own persistent data, background jobs, schedulers, external + subscriptions, or irreversible migration state must expose uninstall guard + providers through their manifest. Guards return `blocker`, `warning`, or + `info` results and may inspect live state through the core-owned DB session. + Default package uninstall is non-destructive, so ordinary persistent data + should warn that data will remain dormant. Guards should block only when + removing the package would corrupt other active modules, workers, external + subscriptions, or deployment state. A guard failure is treated as a blocker. +- Modules that can destroy their own data must also expose a migration + retirement provider. Destructive retirement is opt-in per uninstall plan row + through `destroy_data: true`; the installer then snapshots the database, + invokes the module-owned retirement executor while the package is still + installed, and only then removes Python/WebUI packages. Without that flag, + the same provider is used for preflight reporting only and module tables/data + remain dormant. +- Core refreshes the active registry before frontend metadata is returned from + `/api/v1/platform/modules`; the WebUI shell refetches this metadata after + module changes so navigation, routes, and UI capabilities update without a + page reload. +- Modules can provide `on_activate` and `on_deactivate` hooks for worker, + scheduler, cache, or external subscription lifecycle. These hooks must be + idempotent and must not mutate another module directly. +- Package install/uninstall is performed by the trusted operator installer, not + directly inside FastAPI request handlers. The admin UI can save install plans, + show preflight blockers, and activate/deactivate installed packages. + +The package install-plan API records operator intent only: + +- `GET /api/v1/admin/system/modules/install-plan` reads the saved plan, + renders shell commands, and returns installer preflight status. +- `GET /api/v1/admin/system/modules/install-runs` returns recent installer run + summaries and the current installer lock status. +- `GET /api/v1/admin/system/modules/install-runs/{run_id}` returns the raw run + record for diagnosis. +- `GET /api/v1/admin/system/modules/install-requests` returns daemon handoff + requests queued from the admin UI or CLI plus the current daemon heartbeat. +- `POST /api/v1/admin/system/modules/install-requests` queues a supervised + installer request. It requires maintenance mode and maintenance access. The + FastAPI request writes only the request record; it does not run package + commands. +- `POST /api/v1/admin/system/modules/install-requests/{request_id}/cancel` + cancels a queued request. Running requests are not interrupted by the API; + they remain owned by the installer daemon. +- `POST /api/v1/admin/system/modules/install-requests/{request_id}/retry` + queues a new request using the options from a failed or cancelled request. +- `GET /api/v1/admin/system/modules/package-catalog` reads approved package + references from `GOVOPLAN_MODULE_PACKAGE_CATALOG` so operators can add known + module refs to the install plan without typing them manually. The endpoint + also reports catalog validity, channel, signature, trust state, and the + configured path. +- `POST /api/v1/admin/system/modules/install-plan/catalog/{module_id}` saves + a planned install row from a validated catalog entry. Catalog signature and + approved-channel policy are enforced before the row is saved. +- `POST /api/v1/admin/system/modules/{module_id}/uninstall-plan` saves a + planned non-destructive uninstall row for an installed module after it has + been disabled. The Python distribution name is resolved from the installed + `govoplan.modules` entry point; the WebUI package name comes from the module + manifest. Operators can then edit the saved plan row and set `destroy_data` + when they explicitly want module-owned tables/data retired before package + removal. +- `PUT /api/v1/admin/system/modules/install-plan` saves planned install or + uninstall rows. Install rows must use tagged package or git references, not + local `file:`/workspace paths. Python install rows must also include the + distribution package name so rollback can uninstall newly added packages. +- `DELETE /api/v1/admin/system/modules/install-plan` clears the plan. +- `govoplan-module-install-plan --format shell` or + `python -m govoplan_core.commands.module_install_plan --format shell` renders + the same commands from a server shell. +- `govoplan-module-installer --format shell` runs the same preflight checks from + the server shell. +- `govoplan-module-installer --apply --build-webui` executes the saved plan + after preflight passes, snapshots `pip freeze` and WebUI package files, writes + a run record under the runtime installer directory, and marks planned rows as + applied after success. Successful installs are added to saved startup state by + default; successful uninstalls are removed from saved startup state by default. + Use `--no-activate-installed-modules` or + `--keep-uninstalled-modules-in-desired` only for staged rollout workflows. +- `govoplan-module-installer --supervise --migrate --health-url http://127.0.0.1:8000/health --restart-command ''` + is the preferred disruptive-change path. It applies the plan, optionally runs + migrations in a fresh Python process after a fresh-process manifest + verification, runs the restart command if provided, polls health, and + automatically rolls packages back from the run snapshot if commands, + migrations, restart, or health recovery fail. +- `govoplan-module-installer --daemon` runs the request executor. It polls the + runtime request queue, claims one request at a time, and executes the same + supervised installer flow. `--daemon-once` processes at most one queued + request and exits, which is useful for tests or process-manager one-shot + units. The daemon writes `daemon.status.json` under the installer runtime + directory so the admin UI and CLI can report heartbeat/status. +- `govoplan-module-installer --enqueue-supervised` creates the same request + record from a shell instead of from the admin UI. +- `govoplan-module-installer --daemon-status --format json` reports the daemon + heartbeat. `--cancel-request ` and `--retry-request ` + provide shell equivalents for the admin UI request controls. +- `govoplan-module-installer --validate-package-catalog [path] --format json` + validates a catalog file or the catalog configured through + `GOVOPLAN_MODULE_PACKAGE_CATALOG`. +- `--sign-package-catalog --catalog-signing-key-id --catalog-signing-private-key ` + signs a catalog with Ed25519. `--require-signed-catalog`, + `--approved-catalog-channel `, and + `--catalog-trusted-key =` enforce the approved + release-channel path from an operator shell. +- `govoplan-module-installer --rollback ` restores the saved package + snapshots, restores the captured SQLite or external database snapshot when + present, restores the previous desired module state when the database was not + restored wholesale, and reruns package installation from the previous freeze + file. +- `--database-backup-command ''` and + `--database-restore-command ''` provide non-SQLite backup/restore + hooks for migrated installer runs. The backup hook runs before migrations; + the restore hook runs during rollback and can be overridden on the rollback + command line. +- `govoplan-module-installer --list-runs --format json`, + `--show-run --format json`, and `--lock-status --format json` + expose the same run-history and lock information from the operator shell. +- `--list-requests --format json` and `--show-request --format json` + expose daemon handoff records from the operator shell. + +The supervisor accepts multiple restart commands and health URLs. This is the +process boundary for web, worker, scheduler, and auxiliary service restarts: +each restart command is executed, each health URL is polled, and rollback uses +the same restart/health set after restoring package and database snapshots. + +The installer preflight is intentionally conservative: + +- maintenance mode must be active; +- installed module manifests must be compatible with the supported manifest + contract and current core version; +- uninstalling `tenancy`, `access`, or `admin` is blocked; +- uninstalling an active module is blocked; +- uninstalling a module still present in desired startup state is blocked; +- uninstalling a module with active/desired dependents is blocked; +- uninstalling a module that owns migrations is non-destructive by default: + schema/data remain dormant and preflight emits a warning; +- modules that declare explicit migration retirement support should also + register a retirement provider. Without a provider, preflight emits a + manual-review warning. Providers may return blockers, warnings, and an + explanatory summary, but core does not drop schema or data on behalf of + modules. +- module-owned uninstall guard providers can veto data/migration/worker unsafe + removals; +- install refs must be exact versions or tagged git refs; +- Python install rows must include the package distribution name for rollback; +- WebUI package changes require a WebUI root and trigger rebuild/reload status. + +The installer supervisor must run outside the FastAPI server process. A server +request handler cannot reliably restart or roll back the process that is +currently executing the request. The admin UI therefore remains an operator +planning and request-submission surface; the trusted daemon/CLI is the executor. + +Automatic rollback covers Python and WebUI package state. The manual/shell +daemon can run `npm install` and `npm run build`; this remains the supported +WebUI package path. Browser-loaded remote module bundles are experimental and +reserved for controlled deployments with integrity/signature policy. + +For `sqlite:///` database URLs, `--migrate` also captures a SQLite backup and +rollback restores it before the supervisor restarts the server. For non-SQLite +database URLs, `--migrate` requires deployment-specific backup and restore +commands. A `--database-restore-check-command` can validate the created backup +artifact before migrations proceed. Hook commands run in the installer run +directory with: + +- `GOVOPLAN_INSTALLER_RUN_DIR` +- `GOVOPLAN_DATABASE_URL` +- `GOVOPLAN_DATABASE_BACKUP_PATH` +- `GOVOPLAN_DATABASE_BACKUP_METADATA` + +Module uninstall does not retire data by default. Package removal leaves +module-owned schema/data dormant. Explicit retirement is a later module-owned +operation guarded by retirement providers. + +The running FastAPI server still reports `package_mutation_supported=false` +because dependency-manager operations are not executed inside request handlers. +The trusted mutation boundary is the operator CLI/daemon. This keeps the +interpreter, npm dependency graph, frontend bundle, migrations, and worker +process set under process-supervisor control. + +Frontend module loading primarily uses the build-time package graph generated by +the core WebUI host. Installing or uninstalling a WebUI package therefore still +uses `npm install` plus a WebUI rebuild/reload by default. Core also exposes an +experimental hot remote-bundle path for modules that are enabled by the backend +but absent from the local WebUI package graph: + +- `FrontendModule.asset_manifest` points at a JSON remote asset manifest. +- `asset_manifest_integrity` is an SRI-style hash for that manifest. +- `asset_manifest_signature` and `asset_manifest_public_key_id` allow the shell + to verify the manifest when a trusted browser key is registered on + `window.__GOVOPLAN_REMOTE_MODULE_KEYS__`. +- The remote asset manifest contract version is `1` and contains `moduleId`, + `entry`, `entryIntegrity`, and optional `moduleExport`. +- The browser fetches and verifies the manifest, fetches and verifies the entry + bundle, imports it dynamically, and then applies backend metadata before + adding the module's routes/nav/capabilities. + +Unsigned/unhashed remote bundles are skipped. This keeps remote loading a +controlled deployment option rather than a replacement for release package +builds. + +## Maintenance Mode + +Maintenance mode is the required operating state for package install/uninstall +and other disruptive system maintenance. + +Core stores maintenance mode in `system_settings.settings.maintenance_mode`. +The public platform status endpoint exposes only the flag and message so the +WebUI can show a clear login-screen notice. Login remains reachable during +maintenance so an operator can sign in. + +Authenticated API access is enforced at the access-principal boundary. When +maintenance mode is enabled, authenticated requests require the system scope +`system:maintenance:access`; otherwise the API returns `503 Service +Unavailable` with a maintenance-mode detail payload. The protected +`system_owner` role grants this through `system:*`. A dedicated +`maintenance_operator` role exists for accounts that should be able to access +the system during maintenance without receiving broad write permissions. + +Changing the maintenance-mode flag requires both system settings write access +and `system:maintenance:access`, so an administrator cannot accidentally enable +a mode they cannot use. + +The first implementation is a platform access gate. It does not replace +database backups, process supervision, migration checks, or external load +balancer maintenance pages. + ## Build And Verification Backend verification from core: ```bash cd /mnt/DATA/git/govoplan-core -./.venv/bin/python -m compileall src/govoplan_core ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign +./.venv/bin/python -m compileall src/govoplan_core ../govoplan-access/src/govoplan_access ../govoplan-admin/src/govoplan_admin ../govoplan-tenancy/src/govoplan_tenancy ../govoplan-policy/src/govoplan_policy ../govoplan-audit/src/govoplan_audit ../govoplan-files/src/govoplan_files ../govoplan-mail/src/govoplan_mail ../govoplan-campaign/src/govoplan_campaign +./.venv/bin/python scripts/check_dependency_boundaries.py ``` Core WebUI host verification: diff --git a/docs/RELEASE_CATALOG_WORKFLOW.md b/docs/RELEASE_CATALOG_WORKFLOW.md new file mode 100644 index 0000000..f02479a --- /dev/null +++ b/docs/RELEASE_CATALOG_WORKFLOW.md @@ -0,0 +1,129 @@ +# Release Catalog Workflow + +GovOPlaN release catalogs are published by `govoplan-web` as static JSON and +verified by `govoplan-core` before installer plans are accepted. + +Private signing keys must stay outside all git repositories. Public keyrings +are published with the website. + +## One-Time Key Setup + +Create the first catalog signing key on the release machine: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +mkdir -p "$KEY_DIR" +./.venv/bin/python scripts/generate-catalog-keypair.py \ + --key-id release-key-1 \ + --private-key "$KEY_DIR/release-key-1.pem" \ + --public-key "$KEY_DIR/release-key-1.pub" \ + --keyring "$KEY_DIR/catalog-keyring.json" +``` + +Keep `release-key-1.pem` private. The generated keyring contains only public +material. + +## Publish Current Release Catalog + +Generate the signed catalog into `govoplan-web`: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +scripts/publish-release-catalog.sh \ + --version 0.1.4 \ + --sequence 202607071340 \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ + --build-web +``` + +This writes: + +- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/channels/stable.json` +- `/mnt/DATA/git/govoplan-web/public/catalogs/v1/keyring.json` + +The wrapper validates the catalog with core using the generated public keyring. + +## Publish After A New Release + +For normal module/core releases, first tag and push the module/core repos. Then +publish the website catalog: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +scripts/publish-release-catalog.sh \ + --version \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ + --build-web \ + --commit \ + --tag \ + --push +``` + +The website tag is `catalog-v`. The public URL is: + +```text +https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json +``` + +The public keyring URL is: + +```text +https://govoplan.add-ideas.de/catalogs/v1/keyring.json +``` + +## Integrated Release Script + +`scripts/push-release-tag.sh` can publish the web catalog after module and core +tags have been pushed: + +```bash +cd /mnt/DATA/git/govoplan-core +KEY_DIR="$HOME/.config/govoplan/release-keys" +scripts/push-release-tag.sh \ + --bump subversion \ + --publish-web-catalog \ + --catalog-signing-key "release-key-1=$KEY_DIR/release-key-1.pem" \ + --build-web-catalog +``` + +Use `--catalog-signing-key` more than once during a key rotation window. The +catalog will contain multiple signatures and the public keyring will include the +corresponding public keys. + +## Consumer Configuration + +On a GovOPlaN installation that should consume the official stable catalog: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.add-ideas.de/catalogs/v1/channels/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true +GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=/srv/govoplan/trust/catalog-keyring.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true +``` + +For production, copy the public keyring into deployment configuration and pin it +locally. Do not rely on a URL-fetched keyring as the only trust root. + +## Core Updates + +`stable.json` includes a top-level `core_release` section for operator/update +tooling. Core is intentionally not listed as a normal module entry, because it +must not be added to saved enabled-module state. Core upgrades should remain an +operator-supervised package update with restart and health checks. + +## Key Rotation + +1. Generate the next private key outside git. +2. Run `publish-release-catalog.sh` with both signing keys. +3. Publish the web catalog/keyring. +4. Roll the new public keyring into installations. +5. Stop signing with the old key after the supported fleet trusts the new key. +6. Mark compromised keys as revoked in the public keyring and publish a higher + sequence catalog signed by a trusted uncompromised key. + diff --git a/docs/RELEASE_DEPENDENCIES.md b/docs/RELEASE_DEPENDENCIES.md index 7992f1d..f3b4094 100644 --- a/docs/RELEASE_DEPENDENCIES.md +++ b/docs/RELEASE_DEPENDENCIES.md @@ -28,11 +28,265 @@ cd /mnt/DATA/git/govoplan-core `requirements-release.txt` pins the module repositories to the release tag. Update those refs when cutting a release: ```text -govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.1 -govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.1 -govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.1 +govoplan-access git@git.add-ideas.de:add-ideas/govoplan-access.git v0.1.4 +govoplan-admin git@git.add-ideas.de:add-ideas/govoplan-admin.git v0.1.4 +govoplan-tenancy git@git.add-ideas.de:add-ideas/govoplan-tenancy.git v0.1.4 +govoplan-policy git@git.add-ideas.de:add-ideas/govoplan-policy.git v0.1.4 +govoplan-audit git@git.add-ideas.de:add-ideas/govoplan-audit.git v0.1.4 +govoplan-files git@git.add-ideas.de:add-ideas/govoplan-files.git v0.1.4 +govoplan-mail git@git.add-ideas.de:add-ideas/govoplan-mail.git v0.1.4 +govoplan-campaign git@git.add-ideas.de:add-ideas/govoplan-campaign.git v0.1.4 +govoplan-calendar git@git.add-ideas.de:add-ideas/govoplan-calendar.git v0.1.4 ``` +## Runtime Module Package Changes + +The admin module manager can hot-enable and hot-disable packages that are +already installed. It does not install or uninstall Python/npm packages from +inside the running server. + +For runtime package changes, create an operator install plan in Admin > System > +Modules. The module manager shows the trusted installer preflight status and +blocks unsafe uninstalls before the operator touches packages. + +Preflight from the server shell: + +```bash +govoplan-module-installer --format shell +``` + +Apply from a controlled operator shell while maintenance mode is active: + +```bash +govoplan-module-installer --apply --build-webui +``` + +For real install/uninstall work, prefer supervised mode with the deployment's +restart command and health endpoint: + +```bash +govoplan-module-installer \ + --supervise \ + --migrate \ + --build-webui \ + --health-url http://127.0.0.1:8000/health \ + --restart-command '' +``` + +To let the admin UI trigger package work without executing pip/npm inside a +FastAPI request, run the installer daemon in a separate operator shell. This is +the preferred development/early-production mode for now because the operator can +watch output, stop before queueing risky changes, and keep restart commands +deployment-specific: + +```bash +govoplan-module-installer \ + --daemon \ + --migrate \ + --build-webui \ + --database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \ + --database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --health-url http://127.0.0.1:8000/health \ + --restart-command '' +``` + +Admin > System > Modules can then queue the saved install plan as a supervised +request. Install rows can be planned directly from the approved package catalog; +uninstall rows are generated from installed, disabled modules so the Python +distribution and WebUI package names do not need to be typed by hand. The +daemon claims one queued request at a time and writes request/run records below +`runtime/module-installer`. For process-manager one-shot usage or tests, use +`--daemon-once`. The daemon also writes +`runtime/module-installer/daemon.status.json`; check it with: + +```bash +govoplan-module-installer --daemon-status --format json +``` + +The installer uses a runtime lock, snapshots `pip freeze` plus WebUI +`package.json`/`package-lock.json`, writes a run record below +`runtime/module-installer/runs`, and marks planned rows as applied only after +all commands succeed. When `--migrate` is used with a `sqlite:///` database URL, +the installer also snapshots the SQLite database with SQLite's backup API before +running migrations. For other database engines, pass external backup/restore +hooks: + +```bash +govoplan-module-installer \ + --supervise \ + --migrate \ + --database-backup-command 'pg_dump --format=custom "$GOVOPLAN_DATABASE_URL" > "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --database-restore-check-command 'pg_restore --list "$GOVOPLAN_DATABASE_BACKUP_PATH" >/dev/null' \ + --database-restore-command 'pg_restore --clean --if-exists --dbname "$GOVOPLAN_DATABASE_URL" "$GOVOPLAN_DATABASE_BACKUP_PATH"' \ + --health-url http://127.0.0.1:8000/health \ + --restart-command '' +``` + +The backup command runs before migrations. The restore-check command validates +the produced backup artifact before migrations proceed, without restoring over +the live database. The restore command is stored in the run record and runs +during rollback unless an override is passed to `--rollback`. + +Supervised mode treats package command failure, migration failure, restart +failure, and health timeout as rollback triggers. It restores the Python/WebUI +package snapshots, re-runs the restart command when supplied, and restores the +saved install plan state so the operator can correct it. The supervisor must +run outside the FastAPI server process; the admin UI saves and validates plans +but does not mutate packages from an HTTP request. + +After a successful install plan, the installer adds installed modules to saved +startup state by default so the restarted server can discover and enable them. +After a successful uninstall plan, the installer removes uninstalled modules +from saved startup state by default. Use +`--no-activate-installed-modules` or +`--keep-uninstalled-modules-in-desired` only for staged rollout workflows that +will update module state separately. + +Uninstall is non-destructive by default. A planned uninstall row can set +`destroy_data: true` to request destructive module retirement. The module must +provide an automated retirement provider, and the installer snapshots the +database before dropping module-owned tables. For SQLite this uses the built-in +snapshot path; for PostgreSQL or another non-SQLite database, provide +`--database-backup-command`, `--database-restore-check-command`, and +`--database-restore-command`. If a destructive run fails during package removal, +the installer restores the database snapshot before returning the failed run +result; supervised restart/health failures also roll back through the normal +supervisor path. + +Package rollback is automatic. SQLite database rollback is automatic for +installer runs that used `--migrate` and captured a database snapshot. +Non-SQLite rollback is automatic when the run used +`--database-backup-command` and `--database-restore-command`; otherwise migrated +non-SQLite runs are blocked before package changes are applied. + +Rollback uses the saved run snapshot: + +```bash +govoplan-module-installer --rollback +govoplan-module-installer --rollback --database-restore-command '' +``` + +Database hook commands run with these environment variables: + +- `GOVOPLAN_INSTALLER_RUN_DIR`: the run snapshot directory +- `GOVOPLAN_DATABASE_URL`: the configured database URL +- `GOVOPLAN_DATABASE_BACKUP_PATH`: a suggested backup artifact path inside the + run directory +- `GOVOPLAN_DATABASE_BACKUP_METADATA`: optional JSON metadata path that backup + commands may write for operator diagnostics + +Avoid embedding secrets directly in commands; prefer environment variables, +service credentials, or deployment-local secret injection. + +Inspect installer history and lock state from the operator shell: + +```bash +govoplan-module-installer --list-runs --format json +govoplan-module-installer --show-run --format json +govoplan-module-installer --lock-status --format json +govoplan-module-installer --list-requests --format json +govoplan-module-installer --show-request --format json +govoplan-module-installer --cancel-request --format json +govoplan-module-installer --retry-request --format json +``` + +Package catalogs can be local files or remote static resources, for example +served by `govoplan-web`. Set `GOVOPLAN_MODULE_PACKAGE_CATALOG` for a local file +or `GOVOPLAN_MODULE_PACKAGE_CATALOG_URL` for a remote catalog matching +`docs/module-package-catalog.example.json`; the admin UI will show those entries +and can save them into the install plan. This keeps the release approval +decision outside the running server while avoiding hand-typed package refs. +Remote catalogs can be cached for offline inspection: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG_URL=https://govoplan.example/catalogs/v1/channels/stable.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE=/srv/govoplan/runtime/catalog-cache/stable.json +``` + +Validate the catalog before handing it to operators: + +```bash +govoplan-module-installer --validate-package-catalog docs/module-package-catalog.example.json --format json +``` + +Release catalogs should be signed, channel-gated, expiring, and sequence +tracked. The supported signing format is an Ed25519 signature over the +canonical catalog JSON object with the `signature` and `signatures` fields +removed. Core accepts the legacy single `signature` field and the newer +`signatures` array used during key rotation. Sign a catalog with: + +```bash +govoplan-module-installer \ + --sign-package-catalog docs/module-package-catalog.example.json \ + --catalog-signing-key-id release-key-1 \ + --catalog-signing-private-key /path/to/ed25519-private.pem +``` + +Validate an approved release catalog with: + +```bash +govoplan-module-installer \ + --validate-package-catalog docs/module-package-catalog.example.json \ + --require-signed-catalog \ + --approved-catalog-channel stable \ + --catalog-trusted-key release-key-1= \ + --format json +``` + +For the admin UI/daemon path, configure the same policy through environment +variables: + +```bash +GOVOPLAN_MODULE_PACKAGE_CATALOG=/path/to/catalog.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE=true +GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS=stable +GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS='{"release-key-1":""}' +GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE=/srv/govoplan/runtime/catalog-sequences.json +GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE=true +``` + +Trusted keys can also be loaded from +`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE`. A URL-backed keyring is +supported for development and tightly controlled deployments through +`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL` plus +`GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE`, but production systems +should pin trusted keys locally. + +Catalog entries can declare `license_features`. If +`GOVOPLAN_LICENSE_ENFORCEMENT=true`, core blocks planning catalog installs +whose required features are not present in the configured offline license: + +```bash +GOVOPLAN_LICENSE_FILE=/srv/govoplan/license.json +GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE=/srv/govoplan/trust/license-keyring.json +GOVOPLAN_LICENSE_ENFORCEMENT=true +``` + +See `docs/CATALOG_TRUST_AND_LICENSING.md` for key rotation, replay protection, +and licensing details. See `docs/RELEASE_CATALOG_WORKFLOW.md` for the concrete +release-machine workflow that generates signing keys and publishes signed +catalogs through `govoplan-web`. Unsigned catalogs remain usable for local +development when signature enforcement is off, but the admin UI labels them as +unsigned. + +Install rows must use tagged package/git refs or registry packages, not local +`file:` or workspace links. The installer daemon can run `npm install` and +`npm run build` for WebUI package changes; that is the supported path. Browser +remote bundles are still experimental and should be treated as a controlled +deployment option, not the normal install/uninstall mechanism. + +Module manifests can declare core compatibility bounds and uninstall guard +providers. Preflight blocks incompatible manifest contracts/core versions, +active modules, desired startup state, protected modules, and active dependents. +Default uninstall is non-destructive: module data and schema remain dormant if +the package is removed. Persistent-data guards therefore warn by default instead +of requiring export/delete. A module that supports explicit data/schema +retirement should register a retirement provider. When `destroy_data` is set on +an uninstall plan row, that provider is allowed to destroy module-owned data +after the installer has captured a database snapshot; otherwise it is used only +for preflight reporting. + ## WebUI Local development uses `webui/package.json`, which may point at sibling module checkouts while active development is happening. @@ -46,7 +300,7 @@ cd webui PATH=/home/zemion/.nvm/versions/node/v22.22.3/bin:$PATH /home/zemion/.nvm/versions/node/v22.22.3/bin/npm run build ``` -The module repositories include root-level npm package manifests so git installs can resolve `@govoplan/files-webui`, `@govoplan/mail-webui`, and `@govoplan/campaign-webui` from repository roots even though their source lives below `webui/src`. +The module repositories include root-level npm package manifests so git installs can resolve `@govoplan/access-webui`, `@govoplan/admin-webui`, `@govoplan/files-webui`, `@govoplan/mail-webui`, `@govoplan/campaign-webui`, and `@govoplan/calendar-webui` from repository roots even though their source lives below `webui/src`. The normal release path is automated by `scripts/push-release-tag.sh`: it bumps or accepts the target version, updates Python/WebUI/module manifest versions, commits/tags/pushes the module repositories first, regenerates `webui/package-lock.release.json`, and then commits/tags/pushes core. If the working tree has already been bumped, pass the current version explicitly: @@ -55,16 +309,45 @@ cd /mnt/DATA/git/govoplan-core scripts/push-release-tag.sh --version 0.1.2 ``` +The script also includes GovOPlaN roadmap/scaffold module repositories that do not yet have package metadata. Those repositories are committed, tagged, and pushed with the same release tag, but they are tag-only until they contain `pyproject.toml`, module manifests, or WebUI packages. Tag-only repositories are not listed in `requirements-release.txt` or `webui/package.release.json`. + +Current tag-only module repositories: + +- `govoplan-addresses` +- `govoplan-appointments` +- `govoplan-cases` +- `govoplan-connectors` +- `govoplan-dms` +- `govoplan-erp` +- `govoplan-fit-connect` +- `govoplan-forms` +- `govoplan-identity-trust` +- `govoplan-idm` +- `govoplan-ledger` +- `govoplan-notifications` +- `govoplan-ops` +- `govoplan-payments` +- `govoplan-portal` +- `govoplan-reporting` +- `govoplan-scheduling` +- `govoplan-search` +- `govoplan-tasks` +- `govoplan-templates` +- `govoplan-workflow` +- `govoplan-xoev` +- `govoplan-xrechnung` +- `govoplan-xta-osci` + ### Release lockfile strategy -The supported release composition currently is the full Multi Seal Mail product: core plus files, mail, and campaign. Keep one committed full-product release lockfile at `webui/package-lock.release.json`, generated from `webui/package.release.json` in a clean release workspace. Development `package-lock.json` may continue to point at local `file:` dependencies. +The supported release composition currently is the full Multi Seal Mail product: core plus access, admin, tenancy, policy, audit, files, mail, campaign, and calendar. Keep one committed full-product release lockfile at `webui/package-lock.release.json`, generated from `webui/package.release.json` in a clean release workspace. Development `package-lock.json` may continue to point at local `file:` dependencies. Frontend module permutations are regression-tested through `GOVOPLAN_WEBUI_MODULE_PACKAGES` and temporary build output, not through committed lockfiles for every possible combination. If a smaller composition becomes a separately shipped product, add an explicit release manifest and lockfile pair for that product, for example `package.release.files-mail.json` and `package-lock.release.files-mail.json`, generated in a clean release workspace from tagged git dependencies. ## Release Checklist - Keep Python package versions, WebUI package versions, and git tags aligned. -- Tag core, files, mail, and campaign repositories together. +- Tag core, access, admin, tenancy, policy, audit, files, mail, campaign, calendar, and scaffold module repositories together. - Update `requirements-release.txt` and `webui/package.release.json` when the release tag changes. - Generate the committed full-product release lockfile from `package.release.json` with `scripts/generate-release-lock.sh`. - Add separate release manifest/lockfile pairs only for module compositions that are shipped as their own products. diff --git a/docs/configuration-package-catalog.example.json b/docs/configuration-package-catalog.example.json new file mode 100644 index 0000000..350cafb --- /dev/null +++ b/docs/configuration-package-catalog.example.json @@ -0,0 +1,46 @@ +{ + "channel": "stable", + "packages": [ + { + "package_id": "govoplan.application-handling.basic", + "name": "Basic application handling", + "version": "0.1.0", + "description": "Portal form, case workflow, task creation, mail notification, payment setup, and access roles for a simple application process.", + "publisher": "ADD ideas", + "category": "workflow", + "artifact_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-configuration-packages.git@application-handling-basic-v0.1.0", + "artifact_sha256": "", + "required_modules": [ + { "module_id": "portal", "version": ">=0.1.0" }, + { "module_id": "forms", "version": ">=0.1.0" }, + { "module_id": "cases", "version": ">=0.1.0" }, + { "module_id": "workflow", "version": ">=0.1.0" }, + { "module_id": "tasks", "version": ">=0.1.0" }, + { "module_id": "templates", "version": ">=0.1.0" }, + { "module_id": "mail", "version": ">=0.1.0" }, + { "module_id": "payments", "version": ">=0.1.0" }, + { "module_id": "access", "version": ">=0.1.0" }, + { "module_id": "audit", "version": ">=0.1.0" } + ], + "required_capabilities": [ + "configuration.provider", + "access.configuration", + "forms.configuration", + "workflow.configuration", + "mail.configuration", + "payments.configuration" + ], + "tags": ["official", "example", "public-service"], + "signature": { + "algorithm": "ed25519", + "key_id": "configuration-release-key-1", + "value": "" + } + } + ], + "signature": { + "algorithm": "ed25519", + "key_id": "configuration-catalog-key-1", + "value": "" + } +} diff --git a/docs/gitea-labels.json b/docs/gitea-labels.json new file mode 100644 index 0000000..fc4691b --- /dev/null +++ b/docs/gitea-labels.json @@ -0,0 +1,398 @@ +[ + { + "name": "type/bug", + "color": "d73a4a", + "description": "A reproducible defect, regression, or incorrect behavior.", + "exclusive": true + }, + { + "name": "type/feature", + "color": "0e8a16", + "description": "New user-visible behavior or platform capability.", + "exclusive": true + }, + { + "name": "type/task", + "color": "1d76db", + "description": "Implementation, maintenance, migration, or operational work.", + "exclusive": true + }, + { + "name": "type/debt", + "color": "fbca04", + "description": "Cleanup, refactoring, risk reduction, or deferred engineering work.", + "exclusive": true + }, + { + "name": "type/docs", + "color": "5319e7", + "description": "Documentation, process, or developer workflow work.", + "exclusive": true + }, + { + "name": "priority/p0", + "color": "b60205", + "description": "Immediate stop-the-line priority.", + "exclusive": true + }, + { + "name": "priority/p1", + "color": "d93f0b", + "description": "High priority for the next focused work window.", + "exclusive": true + }, + { + "name": "priority/p2", + "color": "fbca04", + "description": "Normal planned priority.", + "exclusive": true + }, + { + "name": "priority/p3", + "color": "c2e0c6", + "description": "Low priority or opportunistic cleanup.", + "exclusive": true + }, + { + "name": "status/triage", + "color": "d4c5f9", + "description": "Needs review, ownership, priority, or acceptance criteria.", + "exclusive": true + }, + { + "name": "status/ready", + "color": "0e8a16", + "description": "Ready for implementation.", + "exclusive": true + }, + { + "name": "status/in-progress", + "color": "0052cc", + "description": "Currently being worked.", + "exclusive": true + }, + { + "name": "status/blocked", + "color": "b60205", + "description": "Cannot progress without a decision, dependency, credential, or external change.", + "exclusive": true + }, + { + "name": "status/needs-info", + "color": "f9d0c4", + "description": "Needs clarifying input before implementation can proceed safely.", + "exclusive": true + }, + { + "name": "module/access", + "color": "0e8a16", + "description": "GovOPlaN access, identity, authentication, RBAC, and administration behavior.", + "exclusive": false + }, + { + "name": "module/addresses", + "color": "5319e7", + "description": "GovOPlaN Addresses module behavior or integration.", + "exclusive": false + }, + { + "name": "module/admin", + "color": "006b75", + "description": "GovOPlaN Admin module behavior or integration.", + "exclusive": false + }, + { + "name": "module/appointments", + "color": "d876e3", + "description": "GovOPlaN Appointments module behavior or integration.", + "exclusive": false + }, + { + "name": "module/audit", + "color": "0e8a16", + "description": "GovOPlaN Audit module behavior or integration.", + "exclusive": false + }, + { + "name": "module/calendar", + "color": "bfd4f2", + "description": "GovOPlaN Calendar module behavior or integration.", + "exclusive": false + }, + { + "name": "module/campaign", + "color": "d876e3", + "description": "GovOPlaN campaign module behavior or integration.", + "exclusive": false + }, + { + "name": "module/cases", + "color": "1d76db", + "description": "GovOPlaN Cases module behavior or integration.", + "exclusive": false + }, + { + "name": "module/connectors", + "color": "c2e0c6", + "description": "GovOPlaN Connectors module behavior or integration.", + "exclusive": false + }, + { + "name": "module/core", + "color": "0052cc", + "description": "GovOPlaN core runner, shared primitives, shell, or extension points.", + "exclusive": false + }, + { + "name": "module/dms", + "color": "c5def5", + "description": "GovOPlaN Dms module behavior or integration.", + "exclusive": false + }, + { + "name": "module/erp", + "color": "fef2c0", + "description": "GovOPlaN Erp module behavior or integration.", + "exclusive": false + }, + { + "name": "module/files", + "color": "006b75", + "description": "GovOPlaN files module behavior or integration.", + "exclusive": false + }, + { + "name": "module/fit-connect", + "color": "fbca04", + "description": "GovOPlaN Fit Connect module behavior or integration.", + "exclusive": false + }, + { + "name": "module/forms", + "color": "f9d0c4", + "description": "GovOPlaN Forms module behavior or integration.", + "exclusive": false + }, + { + "name": "module/identity-trust", + "color": "d4c5f9", + "description": "GovOPlaN Identity Trust module behavior or integration.", + "exclusive": false + }, + { + "name": "module/idm", + "color": "0052cc", + "description": "GovOPlaN Idm module behavior or integration.", + "exclusive": false + }, + { + "name": "module/ledger", + "color": "5319e7", + "description": "GovOPlaN Ledger module behavior or integration.", + "exclusive": false + }, + { + "name": "module/mail", + "color": "5319e7", + "description": "GovOPlaN mail module behavior or integration.", + "exclusive": false + }, + { + "name": "module/notifications", + "color": "d876e3", + "description": "GovOPlaN Notifications module behavior or integration.", + "exclusive": false + }, + { + "name": "module/ops", + "color": "0e8a16", + "description": "GovOPlaN Ops module behavior or integration.", + "exclusive": false + }, + { + "name": "module/payments", + "color": "bfd4f2", + "description": "GovOPlaN Payments module behavior or integration.", + "exclusive": false + }, + { + "name": "module/policy", + "color": "d93f0b", + "description": "GovOPlaN Policy module behavior or integration.", + "exclusive": false + }, + { + "name": "module/portal", + "color": "1d76db", + "description": "GovOPlaN Portal module behavior or integration.", + "exclusive": false + }, + { + "name": "module/reporting", + "color": "c2e0c6", + "description": "GovOPlaN Reporting module behavior or integration.", + "exclusive": false + }, + { + "name": "module/search", + "color": "bfdadc", + "description": "GovOPlaN Search module behavior or integration.", + "exclusive": false + }, + { + "name": "module/scheduling", + "color": "d93f0b", + "description": "GovOPlaN Scheduling module behavior or integration.", + "exclusive": false + }, + { + "name": "module/tasks", + "color": "c5def5", + "description": "GovOPlaN Tasks module behavior or integration.", + "exclusive": false + }, + { + "name": "module/templates", + "color": "fef2c0", + "description": "GovOPlaN Templates module behavior or integration.", + "exclusive": false + }, + { + "name": "module/tenancy", + "color": "e4e669", + "description": "GovOPlaN Tenancy module behavior or integration.", + "exclusive": false + }, + { + "name": "module/web", + "color": "bfd4f2", + "description": "GovOPlaN public website, product page, or publication content.", + "exclusive": false + }, + { + "name": "module/workflow", + "color": "f9d0c4", + "description": "GovOPlaN Workflow module behavior or integration.", + "exclusive": false + }, + { + "name": "module/xoev", + "color": "d4c5f9", + "description": "GovOPlaN Xoev module behavior or integration.", + "exclusive": false + }, + { + "name": "module/xrechnung", + "color": "0052cc", + "description": "GovOPlaN Xrechnung module behavior or integration.", + "exclusive": false + }, + { + "name": "module/xta-osci", + "color": "5319e7", + "description": "GovOPlaN Xta Osci module behavior or integration.", + "exclusive": false + }, + { + "name": "area/auth", + "color": "bfd4f2", + "description": "Authentication, sessions, access bootstrap, or login behavior.", + "exclusive": false + }, + { + "name": "area/tenancy", + "color": "bfdadc", + "description": "Tenant boundaries, provisioning, or tenant-scoped data behavior.", + "exclusive": false + }, + { + "name": "area/rbac", + "color": "c5def5", + "description": "Permissions, roles, delegation, or authorization policy.", + "exclusive": false + }, + { + "name": "area/governance", + "color": "fef2c0", + "description": "Governance policy, audit, privacy, retention, or compliance behavior.", + "exclusive": false + }, + { + "name": "area/module-system", + "color": "bfe5bf", + "description": "Module discovery, manifests, capabilities, routing, or optional integrations.", + "exclusive": false + }, + { + "name": "area/migrations", + "color": "e4e669", + "description": "Alembic migrations, schema bootstrap, or persistence evolution.", + "exclusive": false + }, + { + "name": "area/webui", + "color": "1d76db", + "description": "Shared WebUI shell, frontend components, routing, or frontend tests.", + "exclusive": false + }, + { + "name": "area/api", + "color": "5319e7", + "description": "HTTP API contracts, routers, schemas, or API smoke behavior.", + "exclusive": false + }, + { + "name": "area/db", + "color": "0e8a16", + "description": "Database sessions, models, transactions, or persistence primitives.", + "exclusive": false + }, + { + "name": "area/devex", + "color": "bfd4f2", + "description": "Local developer workflow, scripts, tests, tooling, or release helpers.", + "exclusive": false + }, + { + "name": "area/release", + "color": "c2e0c6", + "description": "Versioning, release locks, tags, packaging, or dependency pins.", + "exclusive": false + }, + { + "name": "area/docs", + "color": "5319e7", + "description": "Durable documentation and project guidance.", + "exclusive": false + }, + { + "name": "area/marketing", + "color": "f9d0c4", + "description": "Public website, product messaging, publication copy, or legal page content.", + "exclusive": false + }, + { + "name": "source/todo-scan", + "color": "cccccc", + "description": "Imported from inline TODO/FIXME/HACK markers by the Gitea TODO importer.", + "exclusive": false + }, + { + "name": "source/backlog-import", + "color": "bfdadc", + "description": "Imported from markdown backlog, roadmap, plan, or TODO files.", + "exclusive": false + }, + { + "name": "codex/ready", + "color": "0e8a16", + "description": "Suitable for Codex to pick up with the existing issue context.", + "exclusive": false + }, + { + "name": "codex/needs-human", + "color": "f9d0c4", + "description": "Needs an explicit human decision before Codex should implement.", + "exclusive": false + } +] diff --git a/docs/module-package-catalog.example.json b/docs/module-package-catalog.example.json new file mode 100644 index 0000000..3d62630 --- /dev/null +++ b/docs/module-package-catalog.example.json @@ -0,0 +1,47 @@ +{ + "catalog_version": "1", + "channel": "stable", + "sequence": 14, + "generated_at": "2026-07-07T00:00:00Z", + "expires_at": "2030-12-31T23:59:59Z", + "modules": [ + { + "module_id": "files", + "name": "Files", + "description": "Managed file spaces and campaign attachment integration.", + "version": "0.1.4", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4", + "webui_package": "@govoplan/files-webui", + "webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4", + "license_features": ["module.files"], + "tags": ["official", "service-module"] + }, + { + "module_id": "mail", + "name": "Mail", + "description": "SMTP/IMAP profile management and read-only mailbox access.", + "version": "0.1.4", + "action": "install", + "python_package": "govoplan-mail", + "python_ref": "govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4", + "webui_package": "@govoplan/mail-webui", + "webui_ref": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4", + "license_features": ["module.mail"], + "tags": ["official", "service-module"] + } + ], + "signatures": [ + { + "algorithm": "ed25519", + "key_id": "release-key-1", + "value": "" + }, + { + "algorithm": "ed25519", + "key_id": "release-key-2", + "value": "" + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index b57f85e..61ea371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-core" -version = "0.1.4" +version = "0.1.6" description = "Reusable GovOPlaN platform core, access, tenancy, and RBAC components." readme = "README.md" requires-python = ">=3.12" @@ -29,9 +29,8 @@ govoplan_core = ["py.typed"] [project.scripts] govoplan-devserver = "govoplan_core.devserver:main" - -[project.entry-points."govoplan.modules"] -access = "govoplan_core.access.manifest:get_manifest" +govoplan-module-install-plan = "govoplan_core.commands.module_install_plan:main" +govoplan-module-installer = "govoplan_core.commands.module_installer:main" [project.optional-dependencies] server = [ diff --git a/requirements-dev.txt b/requirements-dev.txt index ea5510e..850259f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,11 @@ -e .[server] +-e ../govoplan-tenancy +-e ../govoplan-access +-e ../govoplan-admin +-e ../govoplan-policy +-e ../govoplan-audit -e ../govoplan-files -e ../govoplan-mail -e ../govoplan-campaign +-e ../govoplan-calendar httpx==0.28.1 diff --git a/requirements-release.txt b/requirements-release.txt index 928148e..ed09021 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1,6 +1,12 @@ # Release install from tagged module repositories. # Update GOVOPLAN_RELEASE_TAG together with pyproject/package versions when cutting a release. .[server] -govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.1 -govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.1 -govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.1 +govoplan-tenancy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-tenancy.git@v0.1.4 +govoplan-access @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git@v0.1.4 +govoplan-admin @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git@v0.1.4 +govoplan-policy @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-policy.git@v0.1.4 +govoplan-audit @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-audit.git@v0.1.4 +govoplan-files @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git@v0.1.4 +govoplan-mail @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git@v0.1.4 +govoplan-campaign @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git@v0.1.4 +govoplan-calendar @ git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git@v0.1.4 diff --git a/scripts/check-focused.sh b/scripts/check-focused.sh index 73eea1c..f25afd8 100644 --- a/scripts/check-focused.sh +++ b/scripts/check-focused.sh @@ -17,6 +17,11 @@ import sys roots = [ pathlib.Path("/mnt/DATA/git/govoplan-core/src"), + pathlib.Path("/mnt/DATA/git/govoplan-access/src"), + pathlib.Path("/mnt/DATA/git/govoplan-admin/src"), + pathlib.Path("/mnt/DATA/git/govoplan-tenancy/src"), + pathlib.Path("/mnt/DATA/git/govoplan-policy/src"), + pathlib.Path("/mnt/DATA/git/govoplan-audit/src"), pathlib.Path("/mnt/DATA/git/govoplan-mail/src"), pathlib.Path("/mnt/DATA/git/govoplan-files/src"), pathlib.Path("/mnt/DATA/git/govoplan-campaign/src"), @@ -43,6 +48,7 @@ print(f"AST syntax check passed for {count} Python files") PY ./.venv/bin/python -c 'import govoplan_core.db.bootstrap; import govoplan_core.admin.service; import govoplan_files.backend.router; import govoplan_mail.backend.sending.imap; print("targeted backend imports passed")' +./.venv/bin/python scripts/check_dependency_boundaries.py ./.venv/bin/python -m unittest tests.test_module_system ./.venv/bin/python -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests ./.venv/bin/python -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count diff --git a/scripts/check_dependency_boundaries.py b/scripts/check_dependency_boundaries.py new file mode 100644 index 0000000..ef71dd9 --- /dev/null +++ b/scripts/check_dependency_boundaries.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import ast +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPOS = { + "core": ROOT / "src" / "govoplan_core", + "access": ROOT.parent / "govoplan-access" / "src" / "govoplan_access", + "admin": ROOT.parent / "govoplan-admin" / "src" / "govoplan_admin", + "tenancy": ROOT.parent / "govoplan-tenancy" / "src" / "govoplan_tenancy", + "policy": ROOT.parent / "govoplan-policy" / "src" / "govoplan_policy", + "audit": ROOT.parent / "govoplan-audit" / "src" / "govoplan_audit", + "files": ROOT.parent / "govoplan-files" / "src" / "govoplan_files", + "mail": ROOT.parent / "govoplan-mail" / "src" / "govoplan_mail", + "campaign": ROOT.parent / "govoplan-campaign" / "src" / "govoplan_campaign", +} +PREFIXES = { + "core": "govoplan_core", + "access": "govoplan_access", + "admin": "govoplan_admin", + "tenancy": "govoplan_tenancy", + "policy": "govoplan_policy", + "audit": "govoplan_audit", + "files": "govoplan_files", + "mail": "govoplan_mail", + "campaign": "govoplan_campaign", +} +FEATURE_OWNERS = ("files", "mail", "campaign") +PLATFORM_OWNERS = ("admin", "tenancy", "policy", "audit") +PUBLIC_ACCESS_IMPORTS = ("govoplan_access.backend.auth.dependencies",) + + +@dataclass(frozen=True) +class AllowlistedImport: + owner: str + relative_path: str + imported_owner: str + reason: str + + +# Transitional exceptions. This should remain empty; add entries only with a +# dated removal plan and a capability/API contract issue. +ALLOWLIST: tuple[AllowlistedImport, ...] = () +ALLOWLIST_KEYS = {(item.owner, item.relative_path, item.imported_owner) for item in ALLOWLIST} + + +@dataclass(frozen=True) +class Violation: + owner: str + path: Path + lineno: int + imported: str + imported_owner: str + + +def _import_owner(module_name: str) -> str | None: + for owner, prefix in PREFIXES.items(): + if module_name == prefix or module_name.startswith(f"{prefix}."): + return owner + return None + + +def _imports(path: Path) -> list[tuple[int, str]]: + tree = ast.parse(path.read_text(), filename=str(path)) + result: list[tuple[int, str]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + result.append((node.lineno, alias.name)) + elif isinstance(node, ast.ImportFrom) and node.module: + result.append((node.lineno, node.module)) + return result + + +def _relative(owner: str, path: Path) -> str: + return path.relative_to(REPOS[owner]).as_posix() + + +def _allowed(owner: str, path: Path, imported_owner: str) -> bool: + return (owner, _relative(owner, path), imported_owner) in ALLOWLIST_KEYS + + +def _is_public_access_import(imported: str) -> bool: + return any(imported == item or imported.startswith(f"{item}.") for item in PUBLIC_ACCESS_IMPORTS) + + +def _violations() -> list[Violation]: + violations: list[Violation] = [] + for owner, root in REPOS.items(): + if not root.exists(): + continue + for path in root.rglob("*.py"): + if "__pycache__" in path.parts: + continue + for lineno, imported in _imports(path): + imported_owner = _import_owner(imported) + if imported_owner is None or imported_owner == owner: + continue + if imported_owner == "access" and _is_public_access_import(imported): + continue + if owner == "core" and imported_owner in FEATURE_OWNERS: + if not _allowed(owner, path, imported_owner): + violations.append(Violation(owner, path, lineno, imported, imported_owner)) + elif owner == "access" and imported_owner in FEATURE_OWNERS: + violations.append(Violation(owner, path, lineno, imported, imported_owner)) + elif owner in PLATFORM_OWNERS and imported_owner == "access": + if not _allowed(owner, path, imported_owner): + violations.append(Violation(owner, path, lineno, imported, imported_owner)) + elif owner in FEATURE_OWNERS and imported_owner in FEATURE_OWNERS: + if not _allowed(owner, path, imported_owner): + violations.append(Violation(owner, path, lineno, imported, imported_owner)) + elif owner in FEATURE_OWNERS and imported_owner == "access": + violations.append(Violation(owner, path, lineno, imported, imported_owner)) + return violations + + +def main() -> int: + violations = _violations() + if not violations: + print(f"Dependency boundary check passed ({len(ALLOWLIST)} transitional exceptions tracked)") + return 0 + + print("Dependency boundary violations:") + for item in violations: + print( + f"- {item.path}:{item.lineno}: {item.owner} imports {item.imported} " + f"({item.imported_owner}); use kernel capability/API/event contracts instead" + ) + print("\nIf this is unavoidable transitional debt, add a reasoned ALLOWLIST entry.") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate-catalog-keypair.py b/scripts/generate-catalog-keypair.py new file mode 100644 index 0000000..6b29698 --- /dev/null +++ b/scripts/generate-catalog-keypair.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Generate an Ed25519 keypair for signing GovOPlaN release catalogs.""" + +from __future__ import annotations + +import argparse +import base64 +from datetime import UTC, datetime +import json +from pathlib import Path +import stat + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--key-id", required=True, help="Public key id recorded in catalog signatures.") + parser.add_argument("--private-key", type=Path, required=True, help="Private PEM output path. Keep outside git.") + parser.add_argument("--public-key", type=Path, help="Optional base64 public key output path.") + parser.add_argument("--keyring", type=Path, help="Optional public keyring JSON output path.") + parser.add_argument("--status", default="active", choices=("active", "next", "retired", "revoked", "disabled")) + parser.add_argument("--force", action="store_true", help="Overwrite existing key files.") + args = parser.parse_args() + + private_path = args.private_key.expanduser() + public_path = args.public_key.expanduser() if args.public_key else None + keyring_path = args.keyring.expanduser() if args.keyring else None + if private_path.exists() and not args.force: + raise SystemExit(f"Private key already exists: {private_path}") + if public_path is not None and public_path.exists() and not args.force: + raise SystemExit(f"Public key already exists: {public_path}") + + private_key = Ed25519PrivateKey.generate() + private_bytes = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + public_bytes = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + public_base64 = base64.b64encode(public_bytes).decode("ascii") + + private_path.parent.mkdir(parents=True, exist_ok=True) + private_path.write_bytes(private_bytes) + private_path.chmod(stat.S_IRUSR | stat.S_IWUSR) + + if public_path is not None: + public_path.parent.mkdir(parents=True, exist_ok=True) + public_path.write_text(public_base64 + "\n", encoding="utf-8") + + if keyring_path is not None: + keyring_path.parent.mkdir(parents=True, exist_ok=True) + keyring = { + "keyring_version": "1", + "purpose": "govoplan module package catalog signatures", + "generated_at": datetime.now(tz=UTC).isoformat().replace("+00:00", "Z"), + "keys": [ + { + "key_id": args.key_id, + "status": args.status, + "public_key": public_base64, + "not_before": datetime.now(tz=UTC).date().isoformat() + "T00:00:00Z", + } + ], + } + keyring_path.write_text(json.dumps(keyring, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + print(f"private_key={private_path}") + if public_path is not None: + print(f"public_key={public_path}") + if keyring_path is not None: + print(f"keyring={keyring_path}") + print(f"key_id={args.key_id}") + print(f"public_key_base64={public_base64}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate-release-catalog.py b/scripts/generate-release-catalog.py new file mode 100644 index 0000000..5c8cba3 --- /dev/null +++ b/scripts/generate-release-catalog.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Generate and sign a GovOPlaN module package release catalog.""" + +from __future__ import annotations + +import argparse +import base64 +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +import json +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + +GITEA_BASE = "git+ssh://git@git.add-ideas.de/add-ideas" + + +@dataclass(frozen=True, slots=True) +class CatalogModule: + module_id: str + repo: str + python_package: str + name: str + description: str + tags: tuple[str, ...] + webui_package: str | None = None + + +CATALOG_MODULES = ( + CatalogModule( + module_id="tenancy", + repo="govoplan-tenancy", + python_package="govoplan-tenancy", + name="Tenancy", + description="Tenant registry, tenant settings, and tenant resolution platform module.", + tags=("official", "platform-module"), + ), + CatalogModule( + module_id="access", + repo="govoplan-access", + python_package="govoplan-access", + name="Access", + description="Authentication, accounts, users, groups, roles, API keys, and access capabilities.", + tags=("official", "platform-module"), + webui_package="@govoplan/access-webui", + ), + CatalogModule( + module_id="admin", + repo="govoplan-admin", + python_package="govoplan-admin", + name="Admin", + description="System settings, governance templates, module management, and admin shell contributions.", + tags=("official", "platform-module"), + webui_package="@govoplan/admin-webui", + ), + CatalogModule( + module_id="policy", + repo="govoplan-policy", + python_package="govoplan-policy", + name="Policy", + description="Policy and governance capability module.", + tags=("official", "platform-module"), + ), + CatalogModule( + module_id="audit", + repo="govoplan-audit", + python_package="govoplan-audit", + name="Audit", + description="Audit-log storage and audit administration routes.", + tags=("official", "platform-module"), + ), + CatalogModule( + module_id="files", + repo="govoplan-files", + python_package="govoplan-files", + name="Files", + description="Managed file spaces and campaign attachment integration.", + tags=("official", "service-module"), + webui_package="@govoplan/files-webui", + ), + CatalogModule( + module_id="mail", + repo="govoplan-mail", + python_package="govoplan-mail", + name="Mail", + description="SMTP/IMAP profile management, credential policy, and read-only mailbox access.", + tags=("official", "service-module"), + webui_package="@govoplan/mail-webui", + ), + CatalogModule( + module_id="campaigns", + repo="govoplan-campaign", + python_package="govoplan-campaign", + name="Campaigns", + description="Campaign authoring, validation, queueing, delivery control, and reports.", + tags=("official", "business-module"), + webui_package="@govoplan/campaign-webui", + ), + CatalogModule( + module_id="calendar", + repo="govoplan-calendar", + python_package="govoplan-calendar", + name="Calendar", + description="Calendar collections, events, CalDAV sources, and calendar WebUI routes.", + tags=("official", "service-module"), + webui_package="@govoplan/calendar-webui", + ), +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="GovOPlaN release version, without leading v.") + parser.add_argument("--channel", default="stable") + parser.add_argument("--sequence", type=int, help="Monotonic channel sequence. Defaults to UTC timestamp.") + parser.add_argument("--expires-days", type=int, default=90) + parser.add_argument("--catalog-output", type=Path, required=True) + parser.add_argument("--keyring-output", type=Path) + parser.add_argument( + "--catalog-signing-key", + action="append", + default=[], + metavar="KEY_ID=PRIVATE_KEY", + help="Ed25519 private key used to sign the catalog; may be repeated for rotation.", + ) + parser.add_argument("--public-base-url", default="https://govoplan.add-ideas.de") + parser.add_argument("--repository-base", default=GITEA_BASE) + args = parser.parse_args() + + version = args.version.removeprefix("v") + tag = f"v{version}" + generated_at = datetime.now(tz=UTC) + sequence = args.sequence if args.sequence is not None else int(generated_at.strftime("%Y%m%d%H%M")) + expires_at = generated_at + timedelta(days=args.expires_days) + signing_keys = [_parse_signing_key(value) for value in args.catalog_signing_key] + + catalog = _catalog_payload( + version=version, + tag=tag, + channel=args.channel, + sequence=sequence, + generated_at=generated_at, + expires_at=expires_at, + repository_base=args.repository_base.rstrip("/"), + public_base_url=args.public_base_url.rstrip("/"), + ) + if signing_keys: + catalog["signatures"] = [_signature(catalog, key_id=key_id, private_key=private_key) for key_id, private_key in signing_keys] + + output = args.catalog_output.expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(catalog, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if args.keyring_output is not None: + keyring_output = args.keyring_output.expanduser() + keyring_output.parent.mkdir(parents=True, exist_ok=True) + keyring_output.write_text( + json.dumps(_keyring(signing_keys=signing_keys, generated_at=generated_at), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + print(f"catalog={output}") + if args.keyring_output is not None: + print(f"keyring={args.keyring_output.expanduser()}") + print(f"channel={args.channel}") + print(f"sequence={sequence}") + print(f"version={version}") + return 0 + + +def _catalog_payload( + *, + version: str, + tag: str, + channel: str, + sequence: int, + generated_at: datetime, + expires_at: datetime, + repository_base: str, + public_base_url: str, +) -> dict[str, Any]: + modules: list[dict[str, Any]] = [] + for module in CATALOG_MODULES: + entry: dict[str, Any] = { + "module_id": module.module_id, + "name": module.name, + "description": module.description, + "version": version, + "action": "install", + "python_package": module.python_package, + "python_ref": f"{module.python_package} @ {repository_base}/{module.repo}.git@{tag}", + "license_features": [f"module.{module.module_id}"], + "tags": list(module.tags), + } + if module.webui_package: + entry["webui_package"] = module.webui_package + entry["webui_ref"] = f"{repository_base}/{module.repo}.git#{tag}" + modules.append(entry) + + return { + "catalog_version": "1", + "channel": channel, + "sequence": sequence, + "generated_at": _json_datetime(generated_at), + "expires_at": _json_datetime(expires_at), + "release": { + "version": version, + "tag": tag, + "catalog_url": f"{public_base_url}/catalogs/v1/channels/{channel}.json", + "keyring_url": f"{public_base_url}/catalogs/v1/keyring.json", + }, + "core_release": { + "name": "GovOPlaN Core", + "version": version, + "python_package": "govoplan-core", + "python_ref": f"govoplan-core[server] @ {repository_base}/govoplan-core.git@{tag}", + "webui_package": "@govoplan/core-webui", + "webui_ref": f"{repository_base}/govoplan-core.git#{tag}", + }, + "modules": modules, + } + + +def _parse_signing_key(value: str) -> tuple[str, Ed25519PrivateKey]: + key_id, separator, path_text = value.partition("=") + if not separator or not key_id.strip() or not path_text.strip(): + raise SystemExit("--catalog-signing-key must use KEY_ID=/path/to/private.pem") + path = Path(path_text).expanduser() + private_key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(private_key, Ed25519PrivateKey): + raise SystemExit(f"Catalog signing key must be an Ed25519 private key: {path}") + return key_id.strip(), private_key + + +def _signature(payload: dict[str, Any], *, key_id: str, private_key: Ed25519PrivateKey) -> dict[str, str]: + signature_payload = dict(payload) + signature_payload.pop("signature", None) + signature_payload.pop("signatures", None) + signature = private_key.sign(_canonical_bytes(signature_payload)) + return { + "algorithm": "ed25519", + "key_id": key_id, + "value": base64.b64encode(signature).decode("ascii"), + } + + +def _keyring(*, signing_keys: list[tuple[str, Ed25519PrivateKey]], generated_at: datetime) -> dict[str, Any]: + return { + "keyring_version": "1", + "purpose": "govoplan module package catalog signatures", + "generated_at": _json_datetime(generated_at), + "keys": [ + { + "key_id": key_id, + "status": "active", + "public_key": _public_key_base64(private_key), + "not_before": generated_at.date().isoformat() + "T00:00:00Z", + } + for key_id, private_key in signing_keys + ], + } + + +def _public_key_base64(private_key: Ed25519PrivateKey) -> str: + public_bytes = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + return base64.b64encode(public_bytes).decode("ascii") + + +def _canonical_bytes(payload: object) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _json_datetime(value: datetime) -> str: + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-backlog-import.py b/scripts/gitea-backlog-import.py new file mode 100644 index 0000000..4e122ab --- /dev/null +++ b/scripts/gitea-backlog-import.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 +"""Import markdown backlog/planning files into Gitea issues.""" + +from __future__ import annotations + +import argparse +import dataclasses +import hashlib +import os +import pathlib +import re +import sys +from typing import Any, Iterable + +from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token + + +DEFAULT_PRODUCT_BACKLOG = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products/govoplan/backlog.md") +DEFAULT_ACCESS_PLAN = pathlib.Path("/mnt/DATA/git/govoplan-core/docs/ACCESS_EXTRACTION_PLAN.md") +DEFAULT_WEB_TODO = pathlib.Path("/mnt/DATA/git/govoplan-web/TODO.md") + +REPO_ROOTS = { + "core": pathlib.Path("/mnt/DATA/git/govoplan-core"), + "access": pathlib.Path("/mnt/DATA/git/govoplan-access"), + "mail": pathlib.Path("/mnt/DATA/git/govoplan-mail"), + "files": pathlib.Path("/mnt/DATA/git/govoplan-files"), + "campaign": pathlib.Path("/mnt/DATA/git/govoplan-campaign"), + "web": pathlib.Path("/mnt/DATA/git/govoplan-web"), +} + +MODULE_LABELS = { + "core": "module/core", + "access": "module/access", + "mail": "module/mail", + "files": "module/files", + "campaign": "module/campaign", + "web": "module/web", +} + + +@dataclasses.dataclass(frozen=True) +class Candidate: + repo_key: str + title: str + body: str + labels: tuple[str, ...] + fingerprint: str + source: str + line: int + + +@dataclasses.dataclass +class RepoState: + target: Any + client: GiteaClient + label_ids: dict[str, int] + fingerprints: set[str] + titles: set[str] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values") + parser.add_argument("--product-backlog", type=pathlib.Path, default=DEFAULT_PRODUCT_BACKLOG) + parser.add_argument("--access-plan", type=pathlib.Path, default=DEFAULT_ACCESS_PLAN) + parser.add_argument("--web-todo", type=pathlib.Path, default=DEFAULT_WEB_TODO) + parser.add_argument("--apply", action="store_true", help="create missing Gitea issues") + args = parser.parse_args() + + try: + load_dotenv(args.env_file) + candidates = list(build_candidates(args)) + candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title)) + + print(f"Prepared {len(candidates)} backlog issue candidate(s).") + for repo_key, grouped in group_candidates(candidates).items(): + print(f"{repo_key}: {len(grouped)} candidate(s)") + + token = require_token() if args.apply else os.environ.get("GITEA_TOKEN") + if not token: + print("Dry run without GITEA_TOKEN: duplicate comparison skipped.") + print_preview(candidates) + return 0 + + states = load_repo_states(token, sorted({candidate.repo_key for candidate in candidates})) + missing = [candidate for candidate in candidates if not already_imported(candidate, states[candidate.repo_key])] + skipped = len(candidates) - len(missing) + + print(f"Skipped {skipped} existing issue candidate(s).") + print(f"Missing {len(missing)} issue candidate(s).") + print_preview(missing) + + if not args.apply: + print("Dry run only. Re-run with --apply to create missing issues.") + return 0 + + created_by_repo: dict[str, list[str]] = {} + for candidate in missing: + state = states[candidate.repo_key] + validate_labels(candidate, state) + issue = state.client.request_json( + "POST", + repo_path(state.target.owner, state.target.repo, "/issues"), + body={ + "title": candidate.title, + "body": candidate.body, + "labels": [state.label_ids[label] for label in candidate.labels], + }, + ) + number = issue.get("number") or issue.get("index") + created_by_repo.setdefault(candidate.repo_key, []).append(f"#{number} {candidate.title}") + state.fingerprints.add(candidate.fingerprint) + state.titles.add(normalize_title(candidate.title)) + + print("Created issues:") + for repo_key, titles in created_by_repo.items(): + print(f"{repo_key}:") + for title in titles: + print(f" {title}") + if not created_by_repo: + print(" none") + return 0 + except GiteaError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +def build_candidates(args: argparse.Namespace) -> Iterable[Candidate]: + if args.product_backlog.exists(): + yield from parse_product_backlog(args.product_backlog) + if args.access_plan.exists(): + yield from parse_access_plan(args.access_plan) + if args.web_todo.exists(): + yield from parse_simple_todo(args.web_todo) + + +def parse_product_backlog(path: pathlib.Path) -> Iterable[Candidate]: + headings: list[tuple[int, str]] = [] + status = "" + lines = path.read_text(encoding="utf-8").splitlines() + for index, line in enumerate(lines): + line_number = index + 1 + heading_match = re.match(r"^(#{2,4})\s+(.+?)\s*$", line) + if heading_match: + level = len(heading_match.group(1)) + title = heading_match.group(2).strip() + if level == 2 and title == "Completed Work Archive": + break + headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level] + headings.append((level, title)) + status = "" + continue + + status_match = re.match(r"^\*\*Status:\*\*\s*(.+?)\s*$", line) + if status_match: + status = status_match.group(1).strip() + continue + + item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line) + if not item_match: + continue + + item = collect_continued_item(lines, index, item_match.group(1)) + section = section_title(headings) + repo_key, labels = classify_product_item(item, section) + prefix = type_prefix(labels) + title = format_title(prefix, item) + source = str(path) + body = body_for_item( + fingerprint=make_fingerprint(source, line_number, repo_key, item), + source=source, + line=line_number, + section=section, + status=status, + item=item, + note="Imported from the consolidated GovOPlaN product backlog.", + ) + yield Candidate( + repo_key=repo_key, + title=title, + body=body, + labels=tuple(sorted(set(labels))), + fingerprint=make_fingerprint(source, line_number, repo_key, item), + source=source, + line=line_number, + ) + + +def parse_access_plan(path: pathlib.Path) -> Iterable[Candidate]: + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + yield from access_stage_candidates(path, lines) + yield from access_immediate_candidates(path, lines) + yield from access_decision_candidates(path, lines) + + +def access_stage_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]: + index = 0 + while index < len(lines): + match = re.match(r"^### Stage ([1-7]):\s*(.+?)\s*$", lines[index]) + if not match: + index += 1 + continue + stage = match.group(1) + name = match.group(2).strip() + start_line = index + 1 + index += 1 + block: list[str] = [] + while index < len(lines) and not re.match(r"^### Stage \d+:", lines[index]) and not re.match(r"^## ", lines[index]): + block.append(lines[index]) + index += 1 + item = f"Access extraction Stage {stage}: {name}" + fingerprint = make_fingerprint(str(path), start_line, "core", item) + body = "\n".join( + [ + f"", + "", + "Imported from the GovOPlaN access extraction plan.", + "", + f"- Source: `{path}`", + f"- Line: `{start_line}`", + f"- Section: `Stage {stage}: {name}`", + "", + "Plan excerpt:", + "", + "```markdown", + f"### Stage {stage}: {name}", + *block, + "```", + ] + ) + yield Candidate( + repo_key="core", + title=format_title("[Task]", item), + body=body, + labels=labels("core", "type/task", "area/module-system", "module/access"), + fingerprint=fingerprint, + source=str(path), + line=start_line, + ) + + +def access_immediate_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]: + in_section = False + for index, line in enumerate(lines): + line_number = index + 1 + if line == "## Immediate Backlog": + in_section = True + continue + if in_section and line.startswith("## "): + break + if not in_section: + continue + item_match = re.match(r"^\s*[-*]\s+\[\s\]\s+(.+?)\s*$", line) + if not item_match: + continue + item = collect_continued_item(lines, index, item_match.group(1)) + title_item = access_title_item(item) + fingerprint = make_fingerprint(str(path), line_number, "core", item) + body = body_for_item( + fingerprint=fingerprint, + source=str(path), + line=line_number, + section="Immediate Backlog", + status="open", + item=item, + note="Imported from the GovOPlaN access extraction plan immediate backlog.", + ) + yield Candidate( + repo_key="core", + title=format_title("[Task]", title_item), + body=body, + labels=labels("core", "type/task", "area/auth", "area/module-system", "module/access"), + fingerprint=fingerprint, + source=str(path), + line=line_number, + ) + + +def access_decision_candidates(path: pathlib.Path, lines: list[str]) -> Iterable[Candidate]: + in_section = False + for index, line in enumerate(lines): + line_number = index + 1 + if line == "## Open Decisions": + in_section = True + continue + if in_section and line.startswith("## "): + break + if not in_section: + continue + item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line) + if not item_match: + continue + item = collect_continued_item(lines, index, item_match.group(1)) + fingerprint = make_fingerprint(str(path), line_number, "core", item) + body = body_for_item( + fingerprint=fingerprint, + source=str(path), + line=line_number, + section="Open Decisions", + status="decision needed", + item=item, + note="Imported from the GovOPlaN access extraction plan open decisions.", + ) + yield Candidate( + repo_key="core", + title=format_title("[Task]", f"Decide: {item}"), + body=body, + labels=labels("core", "type/task", "status/needs-info", "codex/needs-human", "area/auth", "module/access"), + fingerprint=fingerprint, + source=str(path), + line=line_number, + ) + + +def parse_simple_todo(path: pathlib.Path) -> Iterable[Candidate]: + lines = path.read_text(encoding="utf-8").splitlines() + for index, line in enumerate(lines): + line_number = index + 1 + item_match = re.match(r"^\s*[-*]\s+(.+?)\s*$", line) + if not item_match: + continue + item = collect_continued_item(lines, index, item_match.group(1)) + fingerprint = make_fingerprint(str(path), line_number, "web", item) + yield Candidate( + repo_key="web", + title=format_title("[Task]", item), + body=body_for_item( + fingerprint=fingerprint, + source=str(path), + line=line_number, + section="TODO", + status="open", + item=item, + note="Imported from the GovOPlaN website TODO file.", + ), + labels=labels("web", "type/task", "area/marketing"), + fingerprint=fingerprint, + source=str(path), + line=line_number, + ) + + +def classify_product_item(item: str, section: str) -> tuple[str, tuple[str, ...]]: + text = f"{section} {item}".lower() + repo_key = "core" + area = "area/devex" + issue_type = "type/task" + + if "dsar search/export/delete" in text: + return "core", labels("core", "type/feature", "area/governance") + if "fully stream zip uploads" in text: + return "files", labels("files", "type/task", "area/api") + if "installer packaging approach" in text: + return "core", labels("core", "type/task", "area/devex") + if "profile reselection/revalidation" in text: + return "campaign", labels("campaign", "type/task", "area/api", "module/mail") + if "module boundaries" in text: + return "core", labels("core", issue_type, "area/module-system") + if "external storage connectors" in text: + repo_key = "files" + area = "area/api" + elif any(phrase in text for phrase in ("report, evidence", "recipient import", "campaign and attachment", "collaboration and advanced governance", "address book, templates")): + repo_key = "campaign" + area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api" + elif "controlled sending" in text: + repo_key = "campaign" + area = "area/api" + elif "mail profiles" in text: + repo_key = "mail" + area = "area/api" + + if any(word in text for word in ("smtp", "imap", "mailbox", "mail profile", "deliverability", "bounce", "openpgp", "s/mime", "pop3", "jmap")): + repo_key = "mail" + area = "area/api" + if any(word in text for word in ("seafile", "nextcloud", "webdav", "smb", "connector", "file rename", "managed storage", "live remote file")): + repo_key = "files" + area = "area/api" + if "external storage connectors" not in text and any( + word in text + for word in ( + "campaign", + "recipient", + "attachment", + "zip", + "evidence", + "wizard", + "send", + "archive", + "message preview", + "address book", + "carddav", + "mailing lists", + ) + ): + repo_key = "campaign" + area = "area/webui" if any(word in text for word in ("page", "table", "display", "ui", "wizard", "editor", "flow", "polish")) else "area/api" + if repo_key == "core" and any(word in text for word in ("session", "device", "auth", "rbac", "role", "tenant", "governance", "policy", "retention", "dsar", "audit", "encryption", "ldap", "oidc", "saml")): + repo_key = "core" + area = "area/auth" if any(word in text for word in ("session", "auth", "ldap", "oidc", "saml")) else "area/governance" + if repo_key == "core" and any(word in text for word in ("module", "manifest", "module boundaries", "installable", "activatable", "deactivatable", "uninstallable")): + repo_key = "core" + area = "area/module-system" + if repo_key == "core" and any(word in text for word in ("release", "version metadata", "lockfile", "package", "tagging")): + repo_key = "core" + area = "area/release" + if repo_key == "core" and not any( + phrase in text + for phrase in ( + "external storage connectors", + "controlled sending", + "report, evidence", + "recipient import", + "campaign and attachment", + ) + ) and any(word in text for word in ("postgres", "redis", "celery", "worker", "deployment", "installer", "backup", "monitoring", "configuration", "dev profile", ".env")): + repo_key = "core" + area = "area/devex" + if any(word in text for word in ("documentation", "handbook", "docs", "runbook")): + area = "area/docs" + if "ui lists often start at index 2" in text: + issue_type = "type/bug" + area = "area/webui" + repo_key = "core" + if any(word in text for word in ("add ", "implement ", "build ", "provide ", "allow ", "make ", "extend ", "replace ")): + issue_type = "type/feature" + if any(word in text for word in ("cleanup", "remove legacy", "fragile", "deferred cleanup")): + issue_type = "type/debt" + + return repo_key, labels(repo_key, issue_type, area) + + +def collect_continued_item(lines: list[str], index: int, initial: str) -> str: + parts = [initial.strip()] + next_index = index + 1 + while next_index < len(lines): + line = lines[next_index] + if not line.startswith((" ", "\t")): + break + stripped = line.strip() + if not stripped: + break + if re.match(r"^[-*]\s+", stripped) or stripped.startswith("#"): + break + parts.append(stripped) + next_index += 1 + return " ".join(parts) + + +def labels(repo_key: str, *extra: str) -> tuple[str, ...]: + base = {"status/triage", "source/backlog-import", "codex/ready", MODULE_LABELS[repo_key]} + base.update(extra) + if "status/needs-info" in base: + base.discard("status/triage") + return tuple(sorted(base)) + + +def type_prefix(label_set: Iterable[str]) -> str: + labels_set = set(label_set) + if "type/bug" in labels_set: + return "[Bug]" + if "type/feature" in labels_set: + return "[Feature]" + if "type/debt" in labels_set: + return "[Debt]" + if "type/docs" in labels_set: + return "[Docs]" + return "[Task]" + + +def format_title(prefix: str, item: str) -> str: + clean = re.sub(r"\s+", " ", item).strip().rstrip(".") + title = f"{prefix} {clean}" + if len(title) <= 180: + return title + return f"{title[:177].rstrip()}..." + + +def access_title_item(item: str) -> str: + lowered = item.lower() + if "compatibility wrappers" in lowered: + return "Wire core auth compatibility through access capabilities" + return item + + +def body_for_item( + *, + fingerprint: str, + source: str, + line: int, + section: str, + status: str, + item: str, + note: str, +) -> str: + lines = [ + f"", + "", + note, + "", + f"- Source: `{source}`", + f"- Line: `{line}`", + f"- Section: `{section}`", + ] + if status: + lines.append(f"- Source status: `{status}`") + lines.extend( + [ + "", + "Imported backlog item:", + "", + "```markdown", + f"- [ ] {item}", + "```", + ] + ) + return "\n".join(lines) + + +def section_title(headings: list[tuple[int, str]]) -> str: + return " > ".join(title for _, title in headings) + + +def make_fingerprint(source: str, line: int, repo_key: str, item: str) -> str: + stable = "\n".join([source, str(line), repo_key, item]) + return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24] + + +def group_candidates(candidates: list[Candidate]) -> dict[str, list[Candidate]]: + grouped: dict[str, list[Candidate]] = {} + for candidate in candidates: + grouped.setdefault(candidate.repo_key, []).append(candidate) + return grouped + + +def load_repo_states(token: str, repo_keys: list[str]) -> dict[str, RepoState]: + states: dict[str, RepoState] = {} + for repo_key in repo_keys: + root = repo_root(REPO_ROOTS[repo_key]) + target = infer_target(root) + client = GiteaClient(target, token) + labels_payload = client.paginate(repo_path(target.owner, target.repo, "/labels")) + label_ids = {str(label["name"]): int(label["id"]) for label in labels_payload if "name" in label and "id" in label} + issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=100) + fingerprints: set[str] = set() + titles: set[str] = set() + for issue in issues: + body = str(issue.get("body") or "") + if issue.get("state") == "closed" and "Moved to `" in body: + continue + titles.add(normalize_title(str(issue.get("title") or ""))) + fingerprints.update(re.findall(r"codex-backlog-fingerprint:([0-9a-f]{24})", body)) + states[repo_key] = RepoState(target=target, client=client, label_ids=label_ids, fingerprints=fingerprints, titles=titles) + return states + + +def already_imported(candidate: Candidate, state: RepoState) -> bool: + return candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles + + +def normalize_title(title: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip() + + +def validate_labels(candidate: Candidate, state: RepoState) -> None: + missing = sorted(label for label in candidate.labels if label not in state.label_ids) + if missing: + joined = ", ".join(missing) + raise GiteaError(f"{candidate.repo_key} is missing labels for {candidate.title!r}: {joined}") + + +def print_preview(candidates: list[Candidate], limit: int = 160) -> None: + for repo_key, grouped in group_candidates(candidates).items(): + print(f"{repo_key}:") + for candidate in grouped[:limit]: + print(f" {candidate.title}") + if len(grouped) > limit: + print(f" ... {len(grouped) - limit} more") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-codex-note.py b/scripts/gitea-codex-note.py new file mode 100644 index 0000000..c49ee02 --- /dev/null +++ b/scripts/gitea-codex-note.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Post a standardized Codex state update comment to a Gitea issue.""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import sys + +from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token + + +STATUS_LABELS = { + "started": "status/in-progress", + "progress": "status/in-progress", + "blocked": "status/blocked", + "needs-info": "status/needs-info", + "ready": "status/ready", + "done": "", + "note": "", +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path") + parser.add_argument("--remote", default="origin", help="git remote to use for target inference") + parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values") + parser.add_argument("--issue", type=int, required=True, help="Gitea issue number") + parser.add_argument("--status", choices=sorted(STATUS_LABELS), default="note") + parser.add_argument("--summary", action="append", default=[], help="summary bullet; may be repeated") + parser.add_argument("--changed", action="append", default=[], help="changed file path; may be repeated") + parser.add_argument("--test", action="append", default=[], help="verification command/result; may be repeated") + parser.add_argument("--next", action="append", default=[], help="next step or blocker; may be repeated") + parser.add_argument("--body-file", type=pathlib.Path, help="additional Markdown body to append") + parser.add_argument("--close", action="store_true", help="close the issue after posting the note") + parser.add_argument("--apply", action="store_true", help="post the comment and optional state update") + args = parser.parse_args() + + try: + root = repo_root(args.root) + load_dotenv(args.env_file or root / ".env") + target = infer_target(root, args.remote) + token = require_token() if args.apply else os.environ.get("GITEA_TOKEN") + body = build_body(args) + + print(f"Target: {target.display}") + print(f"Issue: #{args.issue}") + print(body) + + if not args.apply: + print("Dry run only. Re-run with --apply and GITEA_TOKEN to post.") + return 0 + + client = GiteaClient(target, token) + client.request_json( + "POST", + repo_path(target.owner, target.repo, f"/issues/{args.issue}/comments"), + body={"body": body}, + ) + if args.close: + client.request_json( + "PATCH", + repo_path(target.owner, target.repo, f"/issues/{args.issue}"), + body={"state": "closed"}, + ) + print("Posted Codex note.") + return 0 + except GiteaError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +def build_body(args: argparse.Namespace) -> str: + lines = [f"## Codex State: {args.status}", ""] + append_section(lines, "Summary", args.summary) + append_section(lines, "Changed Files", [f"`{item}`" for item in args.changed]) + append_section(lines, "Verification", [f"`{item}`" for item in args.test]) + append_section(lines, "Next / Blocked", args.next) + if STATUS_LABELS[args.status]: + lines.extend(["", f"Suggested status label: `{STATUS_LABELS[args.status]}`"]) + if args.body_file: + lines.extend(["", args.body_file.read_text(encoding="utf-8").strip()]) + return "\n".join(lines).rstrip() + "\n" + + +def append_section(lines: list[str], title: str, items: list[str]) -> None: + if not items: + return + lines.extend([f"### {title}", ""]) + lines.extend(f"- {item}" for item in items) + lines.append("") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-import-all-backlogs.py b/scripts/gitea-import-all-backlogs.py new file mode 100644 index 0000000..0d5f370 --- /dev/null +++ b/scripts/gitea-import-all-backlogs.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +"""Import backlog-like files from git.add-ideas.de repositories into Gitea.""" + +from __future__ import annotations + +import argparse +import csv +import dataclasses +import hashlib +import os +import pathlib +import re +import subprocess +import sys +from typing import Any, Iterable + +from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token + + +GIT_ROOT = pathlib.Path("/mnt/DATA/git") +PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products") +EXCLUDED_FILE_PATTERNS = ( + "/.git/", + "/.venv/", + "/venv/", + "/site-packages/", + "/node_modules/", + "/__pycache__/", + "/dist/", + "/build/", + "/.cache/", + "/.module-test-build/", +) +BACKLOG_NAME_RE = re.compile(r"(backlog|todo|roadmap|plan|issue|milestone|action)", re.IGNORECASE) +ACTION_RE = re.compile( + r"^(add|avoid|build|convert|create|define|delay|design|document|enable|ensure|expand|extract|fetch|fix|finalize|generate|" + r"implement|import|improve|integrate|keep|make|move|optimize|persist|precompute|promote|provide|" + r"publish|query|replace|review|run|seed|serve|split|start|store|support|test|track|use|wire)\b", + re.IGNORECASE, +) +SKIP_SECTION_RE = re.compile( + r"(completed|implemented|current state|recent fixes|recently completed|working assumptions|stable decisions|" + r"available now|done|archive)", + re.IGNORECASE, +) +ACTIVE_SECTION_RE = re.compile( + r"(p0|p1|p2|p3|mvp|milestone|phase|todo|backlog|roadmap|tasks|testing|frontend|backend|routing|" + r"data outputs|future|critical path|next sprint|optimization|hardening|enhancements?)", + re.IGNORECASE, +) + +GENERIC_LABELS = [ + ("type/task", "1d76db", "Implementation, maintenance, migration, or operational work."), + ("type/feature", "0e8a16", "New user-visible behavior or product capability."), + ("type/debt", "fbca04", "Cleanup, refactoring, risk reduction, or deferred engineering work."), + ("type/docs", "5319e7", "Documentation, process, or developer workflow work."), + ("status/triage", "d4c5f9", "Needs review, ownership, priority, or acceptance criteria."), + ("source/backlog-import", "bfdadc", "Imported from markdown, text, CSV, roadmap, backlog, plan, or TODO files."), + ("codex/ready", "0e8a16", "Suitable for Codex to pick up with the existing issue context."), + ("priority/p0", "b60205", "Immediate stop-the-line priority."), + ("priority/p1", "d93f0b", "High priority for the next focused work window."), + ("priority/p2", "fbca04", "Normal planned priority."), + ("priority/p3", "c2e0c6", "Low priority or opportunistic cleanup."), +] + + +@dataclasses.dataclass(frozen=True) +class RepoInfo: + root: pathlib.Path + remote: str + owner: str + repo: str + + @property + def key(self) -> str: + return self.root.name + + +@dataclasses.dataclass(frozen=True) +class SourceFile: + repo_key: str + path: pathlib.Path + source_kind: str + + +@dataclasses.dataclass(frozen=True) +class Candidate: + repo_key: str + title: str + body: str + labels: tuple[str, ...] + priority: str + milestone: str + fingerprint: str + source: str + line: int + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--env-file", type=pathlib.Path, default=pathlib.Path("/home/zemion/.config/gitea/gitea.env")) + parser.add_argument("--git-root", type=pathlib.Path, default=GIT_ROOT) + parser.add_argument("--products-root", type=pathlib.Path, default=PRODUCTS_ROOT) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + + try: + load_dotenv(args.env_file) + repos = discover_hosted_repos(args.git_root) + sources = discover_sources(repos, args.products_root) + candidates = list(build_candidates(sources)) + candidates = dedupe_candidates(candidates) + candidates.sort(key=lambda item: (item.repo_key, item.source, item.line, item.title)) + + print(f"Hosted repositories: {len(repos)}") + print(f"Backlog-like source files: {len(sources)}") + print(f"Issue candidates: {len(candidates)}") + for repo_key, count in grouped_counts(candidates).items(): + print(f" {repo_key}: {count}") + + token = require_token() if args.apply else os.environ.get("GITEA_TOKEN") + if not token: + print_preview(candidates) + print("Dry run without GITEA_TOKEN: duplicate comparison skipped.") + return 0 + + missing_by_repo: dict[str, list[Candidate]] = {} + skipped = 0 + states: dict[str, RepoState] = {} + for repo_key in sorted({candidate.repo_key for candidate in candidates}): + states[repo_key] = load_repo_state(repos[repo_key], token, apply=args.apply) + for candidate in candidates: + state = states[candidate.repo_key] + if candidate.fingerprint in state.fingerprints or normalize_title(candidate.title) in state.titles: + skipped += 1 + continue + missing_by_repo.setdefault(candidate.repo_key, []).append(candidate) + + missing_total = sum(len(items) for items in missing_by_repo.values()) + print(f"Skipped existing: {skipped}") + print(f"Missing candidates: {missing_total}") + print_preview([candidate for items in missing_by_repo.values() for candidate in items]) + + if not args.apply: + print("Dry run only. Re-run with --apply to create missing issues.") + return 0 + + for repo_key, repo_candidates in missing_by_repo.items(): + state = states[repo_key] + for candidate in repo_candidates: + label_ids = [state.label_ids[label] for label in candidate.labels] + if candidate.priority not in candidate.labels: + label_ids.append(state.label_ids[candidate.priority]) + milestone_id = ensure_milestone(state, candidate.milestone) + created = state.client.request_json( + "POST", + repo_path(state.target.owner, state.target.repo, "/issues"), + body={ + "title": candidate.title, + "body": candidate.body, + "labels": label_ids, + "milestone": milestone_id, + }, + ) + number = created.get("number") or created.get("index") + print(f"created {state.target.owner}/{state.target.repo}#{number}: {candidate.title}") + state.fingerprints.add(candidate.fingerprint) + state.titles.add(normalize_title(candidate.title)) + return 0 + except GiteaError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +@dataclasses.dataclass +class RepoState: + target: Any + client: GiteaClient + label_ids: dict[str, int] + milestone_ids: dict[str, int] + fingerprints: set[str] + titles: set[str] + + +def discover_hosted_repos(git_root: pathlib.Path) -> dict[str, RepoInfo]: + repos: dict[str, RepoInfo] = {} + for gitdir in sorted(git_root.glob("*/.git")): + root = gitdir.parent + result = subprocess.run( + ["git", "-C", str(root), "remote", "get-url", "origin"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + remote = result.stdout.strip() + if "git.add-ideas.de" not in remote: + continue + target = infer_target(root) + repos[root.name] = RepoInfo(root=root, remote=remote, owner=target.owner, repo=target.repo) + return repos + + +def discover_sources(repos: dict[str, RepoInfo], products_root: pathlib.Path) -> list[SourceFile]: + sources: list[SourceFile] = [] + for repo in repos.values(): + for path in repo.root.rglob("*"): + if not path.is_file() or not BACKLOG_NAME_RE.search(path.name): + continue + path_text = path.as_posix() + if any(pattern in path_text for pattern in EXCLUDED_FILE_PATTERNS): + continue + if is_excluded_repo_file(path): + continue + if is_text_backlog(path): + sources.append(SourceFile(repo_key=repo.key, path=path, source_kind="repo")) + + product_map = { + "co2api": "emission-api-lib", + "govoplan": "govoplan-core", + "meubility": "meubility-workbench", + "mousehold": "mousehold", + "prvnncr": "prvnncr-server", + } + if products_root.exists(): + for product_dir in sorted(path for path in products_root.iterdir() if path.is_dir()): + repo_key = product_map.get(product_dir.name) + if not repo_key or repo_key not in repos: + continue + for path in product_dir.rglob("*"): + if path.is_file() and BACKLOG_NAME_RE.search(path.name) and is_text_backlog(path): + sources.append(SourceFile(repo_key=repo_key, path=path, source_kind="product")) + + unique: dict[tuple[str, pathlib.Path], SourceFile] = {} + for source in sources: + unique[(source.repo_key, source.path.resolve())] = source + return sorted(unique.values(), key=lambda item: (item.repo_key, str(item.path))) + + +def is_excluded_repo_file(path: pathlib.Path) -> bool: + text = path.as_posix() + name = path.name.lower() + if "/scripts/gitea-" in text or text.endswith("/scripts/gitea_common.py"): + return True + if "testing_plan" in name or "test_plan" in name: + return True + if text.endswith("/docs/GITEA_ISSUES.md"): + return True + if text.endswith("/docs/GOVOPLAN_MODULE_ROADMAP.md"): + return True + if text.endswith("/docs/ACCESS_EXTRACTION_PLAN.md"): + return True + if text.endswith("/govoplan-web/TODO.md"): + return True + return False + + +def is_text_backlog(path: pathlib.Path) -> bool: + if path.suffix.lower() not in {".md", ".txt", ".csv"}: + return False + try: + chunk = path.read_bytes()[:4096] + except OSError: + return False + if b"\x00" in chunk: + return False + return True + + +def build_candidates(sources: list[SourceFile]) -> Iterable[Candidate]: + for source in sources: + suffix = source.path.suffix.lower() + if suffix == ".csv": + yield from parse_csv_source(source) + else: + yield from parse_text_source(source) + + +def parse_csv_source(source: SourceFile) -> Iterable[Candidate]: + with source.path.open("r", encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + for index, row in enumerate(reader, start=2): + story = (row.get("story") or row.get("title") or row.get("name") or "").strip() + if not story: + continue + identifier = (row.get("id") or "").strip() + priority = normalize_priority(row.get("priority") or "") + milestone = (row.get("epic") or "Backlog").strip() + title = format_title("[Feature]", f"{identifier}: {story}" if identifier else story) + fingerprint = make_fingerprint(source.path, index, source.repo_key, story) + body = "\n".join( + [ + f"", + "", + "Imported from a CSV backlog file.", + "", + f"- Source: `{source.path}`", + f"- Line: `{index}`", + f"- Source kind: `{source.source_kind}`", + f"- Milestone: `{milestone}`", + "", + "CSV row:", + "", + "```text", + ", ".join(f"{key}={value}" for key, value in row.items()), + "```", + ] + ) + yield Candidate( + repo_key=source.repo_key, + title=title, + body=body, + labels=("type/feature", "status/triage", "source/backlog-import", "codex/ready"), + priority=priority, + milestone=milestone, + fingerprint=fingerprint, + source=str(source.path), + line=index, + ) + + +def parse_text_source(source: SourceFile) -> Iterable[Candidate]: + try: + lines = source.path.read_text(encoding="utf-8").splitlines() + except UnicodeDecodeError: + lines = source.path.read_text(encoding="latin-1").splitlines() + + headings: list[tuple[int, str]] = [] + in_tasks = False + for index, line in enumerate(lines): + line_number = index + 1 + heading = parse_heading(line) + if heading: + level, title = heading + headings = [(h_level, h_title) for h_level, h_title in headings if h_level < level] + headings.append((level, title)) + in_tasks = False + continue + + if re.match(r"^\s*(tasks|action items|immediate todos?|recommended next sprint)\s*:?\s*$", line, re.IGNORECASE): + in_tasks = True + continue + + item = parse_open_item(line) + if item is None and is_action_context(source.path, headings, in_tasks): + item = parse_plain_action_item(line) + if item is None: + continue + if not item or should_skip_item(item, headings): + continue + + section = section_title(headings) + priority = priority_from_context(section, item) + milestone = milestone_from_context(source.path, section) + prefix = "[Feature]" if ACTION_RE.match(item) else "[Task]" + title = format_title(prefix, item) + fingerprint = make_fingerprint(source.path, line_number, source.repo_key, item) + body = "\n".join( + [ + f"", + "", + "Imported from a backlog-like text file.", + "", + f"- Source: `{source.path}`", + f"- Line: `{line_number}`", + f"- Source kind: `{source.source_kind}`", + f"- Section: `{section or 'none'}`", + "", + "Imported item:", + "", + "```text", + item, + "```", + ] + ) + yield Candidate( + repo_key=source.repo_key, + title=title, + body=body, + labels=("type/feature" if prefix == "[Feature]" else "type/task", "status/triage", "source/backlog-import", "codex/ready"), + priority=priority, + milestone=milestone, + fingerprint=fingerprint, + source=str(source.path), + line=line_number, + ) + + +def parse_heading(line: str) -> tuple[int, str] | None: + markdown = re.match(r"^(#{1,6})\s+(.+?)\s*$", line) + if markdown: + return len(markdown.group(1)), clean_text(markdown.group(2)) + underlined = re.match(r"^(.+?)\s*$", line) + if underlined and line.strip() and len(line.strip()) < 120: + return None + return None + + +def parse_open_item(line: str) -> str | None: + patterns = [ + r"^(?P\s*)[-*]\s+\[\s\]\s+(?P.+?)\s*$", + r"^(?P\s*)[-*]\s+☐\s+(?P.+?)\s*$", + r"^(?P\s*)\d+[.)]\s+☐\s+(?P.+?)\s*$", + r"^(?P\s*)[-*]\s+\[(?!x\]|X\])(?:[^\]]+)\]\s+(?P.+?)\s*$", + ] + for pattern in patterns: + match = re.match(pattern, line) + if match and leading_width(match.group("indent")) == 0: + return clean_text(match.group("text")) + return None + + +def parse_plain_action_item(line: str) -> str | None: + match = re.match(r"^(?P\s*)[-*]\s+(?P.+?)\s*$", line) + if match and leading_width(match.group("indent")) == 0: + text = clean_text(match.group("text")) + if "✅" in text or text.startswith(("✅", "[x]", "[X]")): + return None + if ACTION_RE.match(text) or is_short_noun_task(text): + return text + return None + + +def is_action_context(path: pathlib.Path, headings: list[tuple[int, str]], in_tasks: bool) -> bool: + if in_tasks: + return True + context = section_title(headings) + if not context: + return False + if SKIP_SECTION_RE.search(context): + return False + if ACTIVE_SECTION_RE.search(context): + return True + return path.name.lower() in {"todo.txt", "todo.md"} + + +def should_skip_item(item: str, headings: list[tuple[int, str]]) -> bool: + context = section_title(headings) + if "✅" in item or item.startswith(("✅", "[x]", "[X]")): + return True + if item.endswith((",", ";")): + return True + if SKIP_SECTION_RE.search(context): + return True + if len(item) < 4: + return True + if item.endswith(":") and len(item.split()) <= 6: + return True + return False + + +def is_short_noun_task(text: str) -> bool: + return len(text.split()) <= 8 and not text.endswith(".") and not re.search(r"://", text) + + +def clean_text(text: str) -> str: + text = text.strip() + text = re.sub(r"^☐\s*", "", text) + text = re.sub(r"^✅\s*", "", text) + text = re.sub(r"\s+", " ", text) + return text.strip(" -") + + +def leading_width(indent: str) -> int: + return len(indent.replace("\t", " ")) + + +def section_title(headings: list[tuple[int, str]]) -> str: + return " > ".join(title for _, title in headings) + + +def priority_from_context(section: str, item: str) -> str: + text = f"{section} {item}".lower() + if "p0" in text or "critical path" in text: + return "priority/p0" + if "p1" in text or "immediate" in text or "next sprint" in text: + return "priority/p1" + if "p2" in text: + return "priority/p2" + if "p3" in text or "future" in text or "later" in text: + return "priority/p3" + return "priority/p2" + + +def normalize_priority(value: str) -> str: + value = value.strip().lower() + if value == "p0": + return "priority/p0" + if value == "p1": + return "priority/p1" + if value == "p2": + return "priority/p2" + if value == "p3": + return "priority/p3" + return "priority/p2" + + +def milestone_from_context(path: pathlib.Path, section: str) -> str: + if section: + parts = [part for part in section.split(" > ") if ACTIVE_SECTION_RE.search(part)] + if parts: + return parts[-1][:120] + return path.stem.replace("_", " ").replace("-", " ").title() + + +def format_title(prefix: str, item: str) -> str: + title = f"{prefix} {clean_text(item).rstrip('.')}" + if len(title) <= 180: + return title + return f"{title[:177].rstrip()}..." + + +def make_fingerprint(path: pathlib.Path, line: int, repo_key: str, item: str) -> str: + stable = "\n".join([str(path.resolve()), str(line), repo_key, item]) + return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24] + + +def dedupe_candidates(candidates: list[Candidate]) -> list[Candidate]: + seen: set[tuple[str, str]] = set() + unique: list[Candidate] = [] + for candidate in candidates: + key = (candidate.repo_key, normalize_title(candidate.title)) + if key in seen: + continue + seen.add(key) + unique.append(candidate) + return unique + + +def grouped_counts(candidates: list[Candidate]) -> dict[str, int]: + counts: dict[str, int] = {} + for candidate in candidates: + counts[candidate.repo_key] = counts.get(candidate.repo_key, 0) + 1 + return dict(sorted(counts.items())) + + +def load_repo_state(repo: RepoInfo, token: str, *, apply: bool) -> RepoState: + target = infer_target(repo.root) + client = GiteaClient(target, token) + labels = client.paginate(repo_path(target.owner, target.repo, "/labels"), limit=50) + label_ids = {str(label["name"]): int(label["id"]) for label in labels} + if apply: + for name, color, description in GENERIC_LABELS: + if name in label_ids: + continue + created = client.request_json( + "POST", + repo_path(target.owner, target.repo, "/labels"), + body={"name": name, "color": color, "description": description, "exclusive": name.startswith(("type/", "status/", "priority/"))}, + ) + label_ids[str(created["name"])] = int(created["id"]) + print(f"created label {target.owner}/{target.repo}:{name}") + + missing = [name for name, _, _ in GENERIC_LABELS if name not in label_ids] + if missing and apply: + raise GiteaError(f"{repo.key} missing labels: {', '.join(missing)}. Re-run with --apply to create them.") + + milestones = client.paginate(repo_path(target.owner, target.repo, "/milestones"), query={"state": "all"}, limit=50) + milestone_ids = {str(item["title"]): int(item["id"]) for item in milestones} + issues = client.paginate(repo_path(target.owner, target.repo, "/issues"), query={"state": "all"}, limit=50) + fingerprints: set[str] = set() + titles: set[str] = set() + for issue in issues: + body = str(issue.get("body") or "") + titles.add(normalize_title(str(issue.get("title") or ""))) + fingerprints.update(re.findall(r"codex-(?:generic-)?backlog-fingerprint:([0-9a-f]{24})", body)) + return RepoState(target=target, client=client, label_ids=label_ids, milestone_ids=milestone_ids, fingerprints=fingerprints, titles=titles) + + +def ensure_milestone(state: RepoState, title: str) -> int: + if title in state.milestone_ids: + return state.milestone_ids[title] + created = state.client.request_json( + "POST", + repo_path(state.target.owner, state.target.repo, "/milestones"), + body={"title": title, "state": "open", "description": "Created from imported backlog-like files."}, + ) + state.milestone_ids[title] = int(created["id"]) + print(f"created milestone {state.target.owner}/{state.target.repo}:{title}") + return state.milestone_ids[title] + + +def normalize_title(title: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", title.lower()).strip() + + +def print_preview(candidates: list[Candidate], limit_per_repo: int = 80) -> None: + by_repo: dict[str, list[Candidate]] = {} + for candidate in candidates: + by_repo.setdefault(candidate.repo_key, []).append(candidate) + for repo_key, items in sorted(by_repo.items()): + print(f"{repo_key}:") + for item in items[:limit_per_repo]: + print(f" {item.title} [{item.priority}; {item.milestone}]") + if len(items) > limit_per_repo: + print(f" ... {len(items) - limit_per_repo} more") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-install-workflow.py b/scripts/gitea-install-workflow.py new file mode 100644 index 0000000..8e3d0a8 --- /dev/null +++ b/scripts/gitea-install-workflow.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Install Gitea issue workflow files into one or more repositories.""" + +from __future__ import annotations + +import argparse +import pathlib +import sys + +from gitea_common import GiteaError, repo_root + + +SOURCE_ROOT = pathlib.Path(__file__).resolve().parents[1] +WORKFLOW_FILES = ( + ".gitea/PULL_REQUEST_TEMPLATE.md", + ".gitea/ISSUE_TEMPLATE/bug_report.md", + ".gitea/ISSUE_TEMPLATE/config.yaml", + ".gitea/ISSUE_TEMPLATE/docs_workflow.md", + ".gitea/ISSUE_TEMPLATE/feature_request.md", + ".gitea/ISSUE_TEMPLATE/task.md", + ".gitea/ISSUE_TEMPLATE/tech_debt.md", +) +MODULE_LABEL_BY_REPO = { + "govoplan-core": "module/core", + "govoplan-access": "module/access", + "govoplan-admin": "module/admin", + "govoplan-tenancy": "module/tenancy", + "govoplan-policy": "module/policy", + "govoplan-audit": "module/audit", + "govoplan-mail": "module/mail", + "govoplan-files": "module/files", + "govoplan-campaign": "module/campaign", + "govoplan-web": "module/web", +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("targets", nargs="*", type=pathlib.Path, help="target repository roots or child paths") + parser.add_argument("--module-label", help="module label to write into issue templates; only valid for one target") + parser.add_argument("--include-labels-file", action="store_true", help="also copy docs/gitea-labels.json") + parser.add_argument("--apply", action="store_true", help="write files instead of previewing changes") + args = parser.parse_args() + + try: + target_roots = [repo_root(path) for path in (args.targets or [pathlib.Path.cwd()])] + unique_roots = list(dict.fromkeys(target_roots)) + if args.module_label and len(unique_roots) != 1: + raise GiteaError("--module-label can only be used with a single target") + + rel_paths = list(WORKFLOW_FILES) + if args.include_labels_file: + rel_paths.append("docs/gitea-labels.json") + + for target_root in unique_roots: + module_label = args.module_label or infer_module_label(target_root) + install_files(target_root, rel_paths, module_label, apply=args.apply) + + if not args.apply: + print("Dry run only. Re-run with --apply to write files.") + return 0 + except GiteaError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +def infer_module_label(target_root: pathlib.Path) -> str: + repo_name = target_root.name + try: + repo_name = target_root.resolve().name + except OSError: + pass + label = MODULE_LABEL_BY_REPO.get(repo_name) + if label: + return label + if repo_name.startswith("govoplan-"): + suffix = repo_name.removeprefix("govoplan-") + if suffix: + return f"module/{suffix}" + raise GiteaError(f"cannot infer module label for {target_root}. Use --module-label module/.") + + +def install_files(target_root: pathlib.Path, rel_paths: list[str], module_label: str, *, apply: bool) -> None: + print(f"Target: {target_root} ({module_label})") + for rel_path in rel_paths: + source = SOURCE_ROOT / rel_path + target = target_root / rel_path + if not source.exists(): + raise GiteaError(f"missing workflow source file: {source}") + + content = source.read_text(encoding="utf-8") + content = transform_content(rel_path, content, module_label) + + existing = target.read_text(encoding="utf-8") if target.exists() else None + if existing == content: + print(f"unchanged {rel_path}") + continue + + action = "create" if existing is None else "update" + print(f"{action if apply else 'would ' + action} {rel_path}") + if apply: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +def transform_content(rel_path: str, content: str, module_label: str) -> str: + if rel_path.startswith(".gitea/ISSUE_TEMPLATE/") and rel_path.endswith(".md"): + return content.replace(" - module/core", f" - {module_label}") + return content + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-sync-labels.py b/scripts/gitea-sync-labels.py new file mode 100644 index 0000000..bf9aa8d --- /dev/null +++ b/scripts/gitea-sync-labels.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Synchronize issue labels to a Gitea repository.""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import sys +from typing import Any + +from gitea_common import ( + GiteaClient, + GiteaError, + infer_target, + load_dotenv, + load_json, + repo_path, + repo_root, + require_token, +) + + +DEFAULT_LABELS_FILE = pathlib.Path(__file__).resolve().parents[1] / "docs" / "gitea-labels.json" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path.cwd(), help="repository root or child path") + parser.add_argument("--remote", default="origin", help="git remote to use for target inference") + parser.add_argument("--labels-file", type=pathlib.Path, default=DEFAULT_LABELS_FILE) + parser.add_argument("--env-file", type=pathlib.Path, help="dotenv file to read before using GITEA_* values") + parser.add_argument("--apply", action="store_true", help="create or update labels in Gitea") + args = parser.parse_args() + + try: + root = repo_root(args.root) + load_dotenv(args.env_file or root / ".env") + target = infer_target(root, args.remote) + labels = _load_labels(args.labels_file) + token = require_token() if args.apply else os.environ.get("GITEA_TOKEN") + + print(f"Target: {target.display}") + print(f"Labels file: {args.labels_file}") + + if not args.apply and not token: + print("Dry run without GITEA_TOKEN: API comparison skipped.") + for label in labels: + print(f"would ensure {label['name']} ({label['color']})") + return 0 + + client = GiteaClient(target, token) + existing = _labels_by_name(client, target.owner, target.repo) + creates: list[dict[str, Any]] = [] + updates: list[tuple[dict[str, Any], dict[str, Any]]] = [] + + for label in labels: + current = existing.get(label["name"]) + if current is None: + creates.append(label) + continue + + patch = _diff_label(current, label) + if patch: + updates.append((current, patch)) + + if not creates and not updates: + print("No label changes needed.") + return 0 + + for label in creates: + print(f"{'create' if args.apply else 'would create'} {label['name']}") + if args.apply: + client.request_json( + "POST", + repo_path(target.owner, target.repo, "/labels"), + body=label, + ) + + for current, patch in updates: + print(f"{'update' if args.apply else 'would update'} {current['name']}: {', '.join(sorted(patch))}") + if args.apply: + client.request_json( + "PATCH", + repo_path(target.owner, target.repo, f"/labels/{current['id']}"), + body=patch, + ) + + return 0 + except GiteaError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +def _load_labels(path: pathlib.Path) -> list[dict[str, Any]]: + payload = load_json(path) + if not isinstance(payload, list): + raise GiteaError(f"{path} must contain a JSON list") + + labels: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, item in enumerate(payload, start=1): + if not isinstance(item, dict): + raise GiteaError(f"label #{index} must be an object") + label = { + "name": str(item.get("name", "")).strip(), + "color": str(item.get("color", "")).strip().lstrip("#").lower(), + "description": str(item.get("description", "")).strip(), + "exclusive": bool(item.get("exclusive", False)), + } + if not label["name"]: + raise GiteaError(f"label #{index} is missing name") + if label["name"] in seen: + raise GiteaError(f"duplicate label name: {label['name']}") + if not _is_hex_color(label["color"]): + raise GiteaError(f"{label['name']} has invalid color {label['color']!r}") + seen.add(label["name"]) + labels.append(label) + return labels + + +def _labels_by_name(client: GiteaClient, owner: str, repo: str) -> dict[str, dict[str, Any]]: + labels = client.paginate(repo_path(owner, repo, "/labels")) + return {str(label.get("name")): label for label in labels} + + +def _diff_label(current: dict[str, Any], desired: dict[str, Any]) -> dict[str, Any]: + patch: dict[str, Any] = {} + if _normalize_color(current.get("color")) != desired["color"]: + patch["color"] = desired["color"] + if str(current.get("description") or "").strip() != desired["description"]: + patch["description"] = desired["description"] + if bool(current.get("exclusive", False)) != desired["exclusive"]: + patch["exclusive"] = desired["exclusive"] + return patch + + +def _normalize_color(value: Any) -> str: + return str(value or "").strip().lstrip("#").lower() + + +def _is_hex_color(value: str) -> bool: + if len(value) != 6: + return False + return all(char in "0123456789abcdef" for char in value) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-sync-wiki.py b/scripts/gitea-sync-wiki.py new file mode 100644 index 0000000..0b26731 --- /dev/null +++ b/scripts/gitea-sync-wiki.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +"""Mirror project documentation files into Gitea repository wikis.""" + +from __future__ import annotations + +import argparse +import base64 +import dataclasses +import hashlib +import os +import pathlib +import re +import shutil +import subprocess +import sys +import urllib.parse +from typing import Any + +from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, require_token + + +GIT_ROOT = pathlib.Path("/mnt/DATA/git") +PRODUCTS_ROOT = pathlib.Path("/mnt/DATA/Nextcloud/ADD ideas UG/Products") +MANAGED_MARKER = "", + "", + "This page is generated from repository and product-directory project files.", + "", + ] + + for source in sources: + content = render_wiki_content(source) + changed = put_wiki_page( + client, + target.owner, + target.repo, + source.page_title, + content, + existing_page_names, + overwrite_unmanaged=overwrite_unmanaged, + ) + print(f"{'updated' if changed else 'unchanged'} {target.owner}/{target.repo} wiki:{source.page_title}") + index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`") + + if write_index: + put_wiki_page( + client, + target.owner, + target.repo, + "Codex-Project-Index", + "\n".join(index_lines) + "\n", + existing_page_names, + overwrite_unmanaged=True, + ) + else: + print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync") + + +def sync_repo_wiki_git( + repo: RepoInfo, + sources: list[WikiSource], + *, + cache_dir: pathlib.Path, + overwrite_unmanaged: bool, + commit_message: str, + write_index: bool = True, +) -> None: + target = infer_target(repo.root) + wiki_root = prepare_wiki_checkout(repo, cache_dir=cache_dir) + index_lines = [ + f"# {target.repo} Project Wiki Index", + "", + f"{MANAGED_MARKER}index -->", + "", + "This page is generated from repository and product-directory project files.", + "", + ] + + for source in sources: + content = render_wiki_content(source) + status = write_wiki_page_file( + wiki_root, + source.page_title, + content, + overwrite_unmanaged=overwrite_unmanaged, + ) + print(f"{status} {target.owner}/{target.repo} wiki:{source.page_title}") + index_lines.append(f"- [{source.page_title}]({source.page_title}) - `{source.path}`") + + if write_index: + write_wiki_page_file( + wiki_root, + "Codex-Project-Index", + "\n".join(index_lines) + "\n", + overwrite_unmanaged=True, + ) + else: + print(f"skipped {target.owner}/{target.repo} wiki:Codex-Project-Index during page-limited sync") + if not git_has_changes(wiki_root): + print(f"unchanged {target.owner}/{target.repo} wiki repository") + return + run_git(wiki_root, "add", "-A") + if not git_has_staged_changes(wiki_root): + print(f"unchanged {target.owner}/{target.repo} wiki repository") + return + ensure_git_identity(wiki_root) + run_git(wiki_root, "commit", "-m", commit_message) + run_git(wiki_root, "push") + print(f"pushed {target.owner}/{target.repo} wiki repository") + + +def prepare_wiki_checkout(repo: RepoInfo, *, cache_dir: pathlib.Path) -> pathlib.Path: + remote = wiki_remote_for_repo(repo.root) + cache_dir.mkdir(parents=True, exist_ok=True) + checkout = cache_dir / f"{repo.owner}-{repo.repo}.wiki" + if (checkout / ".git").exists(): + run_git(checkout, "remote", "set-url", "origin", remote) + run_git(checkout, "fetch", "origin") + branch = current_branch(checkout) + if branch: + run_git(checkout, "reset", "--hard", f"origin/{branch}") + else: + run_git(checkout, "reset", "--hard") + run_git(checkout, "clean", "-fd") + return checkout + if checkout.exists(): + shutil.rmtree(checkout) + result = subprocess.run( + ["git", "clone", remote, str(checkout)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode == 0: + return checkout + checkout.mkdir(parents=True, exist_ok=True) + run_git(checkout, "init") + run_git(checkout, "remote", "add", "origin", remote) + run_git(checkout, "checkout", "-b", "master") + return checkout + + +def wiki_remote_for_repo(repo_root_path: pathlib.Path) -> str: + result = subprocess.run( + ["git", "-C", str(repo_root_path), "remote", "get-url", "origin"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise GiteaError(f"Could not read origin remote for {repo_root_path}") + remote = result.stdout.strip() + if remote.endswith(".git"): + return f"{remote[:-4]}.wiki.git" + return f"{remote}.wiki.git" + + +def write_wiki_page_file(wiki_root: pathlib.Path, title: str, content: str, *, overwrite_unmanaged: bool) -> str: + path = wiki_root / f"{wiki_file_stem(title)}.md" + if path.exists(): + current = read_text(path) + if current == content: + return "unchanged" + if MANAGED_MARKER not in current and not overwrite_unmanaged: + return "skipped unmanaged" + path.write_text(content, encoding="utf-8") + return "updated" + + +def wiki_file_stem(title: str) -> str: + return re.sub(r"[/\\]+", "-", title).strip() or "Home" + + +def git_has_changes(root: pathlib.Path) -> bool: + result = subprocess.run( + ["git", "-C", str(root), "status", "--porcelain"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + raise GiteaError(f"git status failed in {root}: {result.stderr.strip()}") + return bool(result.stdout.strip()) + + +def git_has_staged_changes(root: pathlib.Path) -> bool: + result = subprocess.run( + ["git", "-C", str(root), "diff", "--cached", "--quiet"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode == 0: + return False + if result.returncode == 1: + return True + raise GiteaError(f"git diff --cached failed in {root}: {result.stderr.strip()}") + + +def current_branch(root: pathlib.Path) -> str: + result = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--abbrev-ref", "HEAD"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + branch = result.stdout.strip() + if result.returncode != 0 or branch == "HEAD": + return "" + return branch + + +def ensure_git_identity(root: pathlib.Path) -> None: + if not git_config_value(root, "user.email"): + run_git(root, "config", "user.email", "codex@govoplan.local") + if not git_config_value(root, "user.name"): + run_git(root, "config", "user.name", "Codex") + + +def git_config_value(root: pathlib.Path, key: str) -> str: + result = subprocess.run( + ["git", "-C", str(root), "config", "--get", key], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else "" + + +def run_git(root: pathlib.Path, *args: str) -> None: + result = subprocess.run( + ["git", "-C", str(root), *args], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + command = "git -C " + str(root) + " " + " ".join(args) + raise GiteaError(f"{command} failed: {result.stderr.strip() or result.stdout.strip()}") + + +def list_existing_wiki_pages(client: GiteaClient, owner: str, repo: str) -> list[dict[str, Any]]: + try: + return client.paginate(repo_path(owner, repo, "/wiki/pages"), limit=50) + except GiteaError as exc: + if "HTTP 404" in str(exc) and "/wiki/pages" in str(exc): + return [] + raise + + +def render_wiki_content(source: WikiSource) -> str: + raw = read_text(source.path) + fingerprint = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] + header = [ + f"{MANAGED_MARKER}{fingerprint} -->", + "", + f"> Mirrored from `{source.path}`.", + f"> Origin: `{source.origin}`.", + "> Active tasks and changing state belong in Gitea issues; this wiki page is durable project context.", + "", + "---", + "", + ] + if source.path.suffix.lower() == ".csv": + return "\n".join(header + ["```csv", raw.rstrip(), "```", ""]) + return "\n".join(header) + raw.rstrip() + "\n" + + +def put_wiki_page( + client: GiteaClient, + owner: str, + repo: str, + title: str, + content: str, + existing_page_names: dict[str, str], + *, + overwrite_unmanaged: bool, +) -> bool: + body = { + "title": title, + "content_base64": base64.b64encode(content.encode("utf-8")).decode("ascii"), + "message": f"Sync {title} from project files", + } + if title in existing_page_names: + page_name = existing_page_names[title] + current = client.request_json("GET", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}")) + current_content = decode_wiki_content(current) + if current_content == content: + return False + if MANAGED_MARKER not in current_content and not overwrite_unmanaged: + print(f"skip unmanaged existing wiki page {owner}/{repo}:{title}") + return False + client.request_json("PATCH", repo_path(owner, repo, f"/wiki/page/{quote_wiki_page_name(page_name)}"), body=body) + return True + client.request_json("POST", repo_path(owner, repo, "/wiki/new"), body=body) + existing_page_names[title] = title + return True + + +def quote_wiki_page_name(value: str) -> str: + return urllib.parse.quote(value, safe="+") + + +def decode_wiki_content(page: dict[str, Any]) -> str: + encoded = str(page.get("content_base64") or "") + if not encoded: + return "" + return base64.b64decode(encoded).decode("utf-8") + + +def read_text(path: pathlib.Path) -> str: + try: + return path.read_text(encoding="utf-8") + except UnicodeDecodeError: + return path.read_text(encoding="latin-1") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea-todo-import.py b/scripts/gitea-todo-import.py new file mode 100644 index 0000000..1354268 --- /dev/null +++ b/scripts/gitea-todo-import.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Preview or import inline TODO-style markers as Gitea issues.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import subprocess +import sys +from typing import Any + +from gitea_common import GiteaClient, GiteaError, infer_target, load_dotenv, repo_path, repo_root, require_token + + +MARKER_RE = re.compile( + r"(?P#|//|/\*|\*|--|", + "", + "Imported from an inline marker.", + "", + f"- Repository: `{owner}/{repo}`", + f"- Location: `{marker.location}`", + f"- Marker: `{marker.marker}`", + f"- Context: `{marker.context or 'none'}`", + "", + "Current line:", + "", + "```text", + marker.raw, + "```", + "", + "Migration rule:", + "", + "- Keep this Gitea issue as the canonical backlog item.", + "- When touching this code, remove the inline marker or replace it with a short issue reference.", + ] + ) + return IssuePreview(title=title, body=body, labels=labels, fingerprint=fingerprint, location=marker.location) + + +def clean_marker_text(text: str) -> str: + cleaned = text.strip() + cleaned = re.sub(r"\s*(?:#|//|/\*|\*|--)\s*$", "", cleaned).strip() + cleaned = re.sub(r"\s*\*/\s*$", "", cleaned).strip() + return cleaned + + +def default_labels(marker: Marker, *, module_label: str, infer_areas: bool) -> list[str]: + labels = ["source/todo-scan", "status/triage", module_label, marker_type_label(marker.marker)] + area = infer_area_label(marker.path) if infer_areas else "" + if area: + labels.append(area) + return labels + + +def marker_type_label(marker: str) -> str: + if marker == "FIXME": + return "type/bug" + if marker in {"HACK", "XXX"}: + return "type/debt" + return "type/task" + + +def infer_area_label(path: pathlib.Path) -> str: + text = path.as_posix() + if text.startswith("webui/"): + return "area/webui" + if text.startswith("alembic/"): + return "area/migrations" + if text.startswith("docs/"): + return "area/docs" + if text.startswith("scripts/"): + return "area/devex" + if "/rbac" in text or "permission" in text: + return "area/rbac" + if "/access" in text or "auth" in text or "session" in text: + return "area/auth" + if "tenant" in text: + return "area/tenancy" + if "governance" in text or "privacy" in text or "audit" in text or "retention" in text: + return "area/governance" + if "module" in text: + return "area/module-system" + if "migration" in text: + return "area/migrations" + if "router" in text or "api" in text: + return "area/api" + if "db" in text or "model" in text or "persistence" in text: + return "area/db" + return "" + + +def infer_module_label(root: pathlib.Path) -> str: + known = { + "govoplan-core": "module/core", + "govoplan-access": "module/access", + "govoplan-admin": "module/admin", + "govoplan-tenancy": "module/tenancy", + "govoplan-policy": "module/policy", + "govoplan-audit": "module/audit", + "govoplan-mail": "module/mail", + "govoplan-files": "module/files", + "govoplan-campaign": "module/campaign", + "govoplan-web": "module/web", + } + if root.name in known: + return known[root.name] + if root.name.startswith("govoplan-"): + suffix = root.name.removeprefix("govoplan-") + if suffix: + return f"module/{suffix}" + return "module/core" + + +def marker_fingerprint(owner: str, repo: str, marker: Marker) -> str: + stable = "\n".join([owner, repo, marker.path.as_posix(), marker.marker, marker.context, marker.text]) + return hashlib.sha256(stable.encode("utf-8")).hexdigest()[:24] + + +def truncate_title(title: str, limit: int = 180) -> str: + if len(title) <= limit: + return title + return f"{title[: limit - 1].rstrip()}..." + + +def load_label_ids(client: GiteaClient, owner: str, repo: str) -> dict[str, int]: + labels = client.paginate(repo_path(owner, repo, "/labels")) + return {str(label["name"]): int(label["id"]) for label in labels if "name" in label and "id" in label} + + +def load_existing_fingerprints(client: GiteaClient, owner: str, repo: str) -> set[str]: + issues = client.paginate( + repo_path(owner, repo, "/issues"), + query={"state": "all", "labels": "source/todo-scan"}, + ) + fingerprints: set[str] = set() + for issue in issues: + body = str(issue.get("body") or "") + for match in re.finditer(r"(?:codex|govoplan)-todo-fingerprint:([0-9a-f]{24})", body): + fingerprints.add(match.group(1)) + return fingerprints + + +def _already_linked(context: str, raw_line: str) -> bool: + linked_text = f"{context} {raw_line}".lower() + return "gitea#" in linked_text or "issue#" in linked_text or re.search(r"#\d+", linked_text) is not None + + +def _looks_like_marker(match: re.Match[str]) -> bool: + text = (match.group("text") or "").strip() + return bool(match.group("context") or match.group("colon") or (match.group("prefix") and text)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitea_common.py b/scripts/gitea_common.py new file mode 100644 index 0000000..09279f4 --- /dev/null +++ b/scripts/gitea_common.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Shared helpers for Gitea maintenance scripts.""" + +from __future__ import annotations + +import dataclasses +import json +import os +import pathlib +import re +import subprocess +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +class GiteaError(RuntimeError): + """Raised when a Gitea API request fails.""" + + +@dataclasses.dataclass(frozen=True) +class RepoTarget: + base_url: str + owner: str + repo: str + + @property + def display(self) -> str: + return f"{self.base_url.rstrip('/')}/{self.owner}/{self.repo}" + + @property + def api_base(self) -> str: + return f"{self.base_url.rstrip('/')}/api/v1" + + +def repo_root(start: pathlib.Path) -> pathlib.Path: + result = subprocess.run( + ["git", "-C", str(start), "rev-parse", "--show-toplevel"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + return start.resolve() + return pathlib.Path(result.stdout.strip()).resolve() + + +def infer_target(root: pathlib.Path, remote_name: str = "origin") -> RepoTarget: + env_url = os.environ.get("GITEA_URL") + env_owner = os.environ.get("GITEA_OWNER") + env_repo = os.environ.get("GITEA_REPO") + + remote_url = _git_remote(root, remote_name) + inferred_url, inferred_owner, inferred_repo = _parse_remote(remote_url) + + base_url = (env_url or inferred_url).rstrip("/") + owner = env_owner or inferred_owner + repo = env_repo or inferred_repo + + missing = [ + name + for name, value in ( + ("GITEA_URL", base_url), + ("GITEA_OWNER", owner), + ("GITEA_REPO", repo), + ) + if not value + ] + if missing: + joined = ", ".join(missing) + raise GiteaError( + f"Could not infer Gitea target. Set {joined}, or configure git remote {remote_name!r}." + ) + return RepoTarget(base_url=base_url, owner=owner, repo=_strip_git_suffix(repo)) + + +def quote_path(value: str) -> str: + return urllib.parse.quote(value, safe="") + + +class GiteaClient: + def __init__(self, target: RepoTarget, token: str | None) -> None: + self.target = target + self.token = token + + def request_json( + self, + method: str, + path: str, + *, + body: dict[str, Any] | None = None, + query: dict[str, Any] | None = None, + ) -> Any: + url = f"{self.target.api_base}{path}" + if query: + url = f"{url}?{urllib.parse.urlencode(query, doseq=True)}" + + data = None + headers = { + "Accept": "application/json", + "User-Agent": "codex-gitea-maintenance", + } + if self.token: + headers["Authorization"] = f"token {self.token}" + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=30) as response: + payload = response.read().decode("utf-8") + if not payload: + return None + return json.loads(payload) + except urllib.error.HTTPError as exc: + message = exc.read().decode("utf-8", errors="replace") + raise GiteaError(f"{method} {url} failed with HTTP {exc.code}: {message}") from exc + except urllib.error.URLError as exc: + raise GiteaError(f"{method} {url} failed: {exc.reason}") from exc + + def paginate( + self, + path: str, + *, + query: dict[str, Any] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + page = 1 + effective_limit = min(limit, 50) + seen_pages: set[tuple[Any, ...]] = set() + while True: + page_query = dict(query or {}) + page_query.update({"page": page, "limit": effective_limit}) + payload = self.request_json("GET", path, query=page_query) + if not isinstance(payload, list): + raise GiteaError(f"Expected a list from {path}, got {type(payload).__name__}") + if not payload: + return items + signature = tuple( + item.get("id") or item.get("number") or item.get("index") or item.get("name") + for item in payload + if isinstance(item, dict) + ) + if signature in seen_pages: + return items + seen_pages.add(signature) + items.extend(payload) + if len(payload) < effective_limit: + return items + page += 1 + + +def load_json(path: pathlib.Path) -> Any: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def load_dotenv(path: pathlib.Path | None) -> list[str]: + if path is None or not path.exists(): + return [] + + loaded: list[str] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.startswith("export "): + stripped = stripped[len("export ") :].strip() + if "=" not in stripped: + raise GiteaError(f"{path}:{line_number}: expected KEY=value") + key, value = stripped.split("=", 1) + key = key.strip() + value = _strip_env_value(value.strip()) + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + raise GiteaError(f"{path}:{line_number}: invalid environment key {key!r}") + if key not in os.environ: + os.environ[key] = value + loaded.append(key) + return loaded + + +def require_token() -> str: + token = os.environ.get("GITEA_TOKEN") + if not token: + raise GiteaError( + "GITEA_TOKEN is required when using --apply. " + "Set it in the environment, the target .env, or --env-file." + ) + return token + + +def repo_path(owner: str, repo: str, suffix: str) -> str: + return f"/repos/{quote_path(owner)}/{quote_path(repo)}{suffix}" + + +def _git_remote(root: pathlib.Path, remote_name: str) -> str: + result = subprocess.run( + ["git", "-C", str(root), "remote", "get-url", remote_name], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def _parse_remote(remote_url: str) -> tuple[str, str, str]: + if not remote_url: + return "", "", "" + + parsed = urllib.parse.urlparse(remote_url) + if parsed.scheme in {"http", "https", "ssh"} and parsed.netloc: + path_parts = [part for part in parsed.path.strip("/").split("/") if part] + owner, repo = _owner_repo_from_parts(path_parts) + if parsed.scheme in {"http", "https"}: + prefix = "/".join(path_parts[:-2]) + base_path = f"/{prefix}" if prefix else "" + return f"{parsed.scheme}://{parsed.netloc}{base_path}", owner, repo + return f"https://{parsed.hostname or parsed.netloc}", owner, repo + + scp_like = re.match(r"^(?:[^@]+@)?(?P[^:]+):(?P.+)$", remote_url) + if scp_like: + path_parts = [part for part in scp_like.group("path").strip("/").split("/") if part] + owner, repo = _owner_repo_from_parts(path_parts) + return f"https://{scp_like.group('host')}", owner, repo + + return "", "", "" + + +def _owner_repo_from_parts(path_parts: list[str]) -> tuple[str, str]: + if len(path_parts) < 2: + return "", "" + return path_parts[-2], _strip_git_suffix(path_parts[-1]) + + +def _strip_git_suffix(value: str) -> str: + return value[:-4] if value.endswith(".git") else value + + +def _strip_env_value(value: str) -> str: + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + return value diff --git a/scripts/launch-dev.sh b/scripts/launch-dev.sh index 276b0e2..e7a0577 100644 --- a/scripts/launch-dev.sh +++ b/scripts/launch-dev.sh @@ -12,6 +12,10 @@ BACKEND_PORT="${GOVOPLAN_BACKEND_PORT:-8000}" FRONTEND_HOST="${GOVOPLAN_FRONTEND_HOST:-127.0.0.1}" FRONTEND_PORT="${GOVOPLAN_FRONTEND_PORT:-5173}" OPEN_BROWSER="${OPEN_BROWSER:-1}" +FRONTEND_FORCE_RELOAD="${GOVOPLAN_FRONTEND_FORCE_RELOAD:-1}" +FRONTEND_CLEAR_VITE_CACHE="${GOVOPLAN_FRONTEND_CLEAR_VITE_CACHE:-1}" +FRONTEND_USE_POLLING="${GOVOPLAN_FRONTEND_USE_POLLING:-1}" +FRONTEND_POLLING_INTERVAL="${GOVOPLAN_FRONTEND_POLLING_INTERVAL:-500}" LOG_DIR="$ROOT/runtime/dev-launcher" BACKEND_LOG="$LOG_DIR/backend.log" @@ -84,6 +88,10 @@ mkdir -p "$LOG_DIR" : > "$BACKEND_LOG" : > "$FRONTEND_LOG" +if [ "$FRONTEND_CLEAR_VITE_CACHE" = "1" ]; then + rm -rf "$WEBUI_ROOT/node_modules/.vite" "$WEBUI_ROOT/node_modules/.vite-temp" +fi + port_is_free "$BACKEND_HOST" "$BACKEND_PORT" || fail "$BACKEND_URL is already in use" port_is_free "$FRONTEND_HOST" "$FRONTEND_PORT" || fail "$FRONTEND_URL is already in use" @@ -103,9 +111,15 @@ wait_for_url "$BACKEND_URL/health" || { printf 'Starting GovOPlaN WebUI at %s\n' "$FRONTEND_URL" ( cd "$WEBUI_ROOT" + frontend_args=(run dev) + if [ "$FRONTEND_FORCE_RELOAD" = "1" ]; then + frontend_args+=(-- --force) + fi PATH="$WEBUI_ROOT/node_modules/.bin:$NODE_BIN:$PATH" \ + CHOKIDAR_USEPOLLING="$FRONTEND_USE_POLLING" \ + CHOKIDAR_INTERVAL="$FRONTEND_POLLING_INTERVAL" \ VITE_API_PROXY_TARGET="$BACKEND_URL" \ - "$NPM" run dev + "$NPM" "${frontend_args[@]}" ) >"$FRONTEND_LOG" 2>&1 & frontend_pid="$!" @@ -130,6 +144,11 @@ Logs: Backend: $BACKEND_LOG WebUI: $FRONTEND_LOG +Frontend reload: +Vite cache cleared: $FRONTEND_CLEAR_VITE_CACHE +Vite --force: $FRONTEND_FORCE_RELOAD +Watch polling: $FRONTEND_USE_POLLING (${FRONTEND_POLLING_INTERVAL}ms) + Press Ctrl+C to stop both processes. EOF diff --git a/scripts/publish-release-catalog.sh b/scripts/publish-release-catalog.sh new file mode 100644 index 0000000..532fd21 --- /dev/null +++ b/scripts/publish-release-catalog.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + scripts/publish-release-catalog.sh --version --catalog-signing-key [options] + +Generates a signed module package catalog for a GovOPlaN release, writes it into +govoplan-web/public/catalogs/v1, validates it, optionally builds the website, +and optionally commits/tags/pushes govoplan-web. + +Options: + --version Release version without leading v. + --channel Catalog channel. Defaults to stable. + --sequence Monotonic channel sequence. Defaults to UTC timestamp. + --expires-days Catalog expiry window. Defaults to 90. + --catalog-signing-key + Ed25519 private key. May be repeated for rotation. + --core-root govoplan-core checkout. Defaults to this script parent. + --web-root govoplan-web checkout. Defaults to ../govoplan-web. + --public-base-url Public website URL. Defaults to https://govoplan.add-ideas.de. + --npm npm executable used for optional web build. + --build-web Run npm run build in govoplan-web after generating files. + --commit Commit generated catalog files in govoplan-web. + --tag Tag govoplan-web with catalog-v. Implies --commit. + --push Push govoplan-web branch and tag. Implies --commit. + --remote Git remote for push. Defaults to origin. + --branch Branch to push. Defaults to current govoplan-web branch. + -n, --dry-run Print commands and validate inputs without writing git changes. + -h, --help Show this help. +USAGE +} + +fail() { + echo "error: $*" >&2 + exit 1 +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PARENT="$(dirname "$ROOT")" +CORE_ROOT="$ROOT" +WEB_ROOT="$PARENT/govoplan-web" +VERSION="" +CHANNEL="stable" +SEQUENCE="" +EXPIRES_DAYS="90" +PUBLIC_BASE_URL="https://govoplan.add-ideas.de" +REMOTE="origin" +BRANCH="" +BUILD_WEB=0 +COMMIT=0 +TAG_WEB=0 +PUSH=0 +DRY_RUN=0 +SIGNING_KEYS=() +NPM_BIN="${NPM:-}" + +if [[ -z "${PYTHON:-}" ]]; then + if [[ -x "$ROOT/.venv/bin/python" ]]; then + PYTHON="$ROOT/.venv/bin/python" + else + PYTHON="python3" + fi +fi + +if [[ -z "$NPM_BIN" ]]; then + if [[ -x "/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" ]]; then + NPM_BIN="/home/zemion/.nvm/versions/node/v22.22.3/bin/npm" + else + NPM_BIN="npm" + fi +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + [[ $# -ge 2 ]] || fail "missing value for $1" + VERSION="${2#v}" + shift 2 + ;; + --channel) + [[ $# -ge 2 ]] || fail "missing value for $1" + CHANNEL="$2" + shift 2 + ;; + --sequence) + [[ $# -ge 2 ]] || fail "missing value for $1" + SEQUENCE="$2" + shift 2 + ;; + --expires-days) + [[ $# -ge 2 ]] || fail "missing value for $1" + EXPIRES_DAYS="$2" + shift 2 + ;; + --catalog-signing-key) + [[ $# -ge 2 ]] || fail "missing value for $1" + SIGNING_KEYS+=("$2") + shift 2 + ;; + --core-root) + [[ $# -ge 2 ]] || fail "missing value for $1" + CORE_ROOT="$(cd "$2" && pwd)" + shift 2 + ;; + --web-root) + [[ $# -ge 2 ]] || fail "missing value for $1" + WEB_ROOT="$(cd "$2" && pwd)" + shift 2 + ;; + --public-base-url) + [[ $# -ge 2 ]] || fail "missing value for $1" + PUBLIC_BASE_URL="$2" + shift 2 + ;; + --npm) + [[ $# -ge 2 ]] || fail "missing value for $1" + NPM_BIN="$2" + shift 2 + ;; + --build-web) + BUILD_WEB=1 + shift + ;; + --commit) + COMMIT=1 + shift + ;; + --tag) + TAG_WEB=1 + COMMIT=1 + shift + ;; + --push) + PUSH=1 + COMMIT=1 + shift + ;; + --remote) + [[ $# -ge 2 ]] || fail "missing value for $1" + REMOTE="$2" + shift 2 + ;; + --branch) + [[ $# -ge 2 ]] || fail "missing value for $1" + BRANCH="$2" + shift 2 + ;; + -n|--dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +[[ -n "$VERSION" ]] || fail "--version is required" +[[ "$VERSION" =~ ^[0-9]+[.][0-9]+[.][0-9]+$ ]] || fail "version must be x.y.z: $VERSION" +[[ ${#SIGNING_KEYS[@]} -gt 0 ]] || fail "at least one --catalog-signing-key is required" +[[ -d "$CORE_ROOT/.git" ]] || fail "not a govoplan-core git repo: $CORE_ROOT" +[[ -d "$WEB_ROOT/.git" ]] || fail "not a govoplan-web git repo: $WEB_ROOT" +command -v "$PYTHON" >/dev/null 2>&1 || fail "Python not found: $PYTHON" + +if [[ "$BUILD_WEB" -eq 1 ]]; then + command -v "$NPM_BIN" >/dev/null 2>&1 || fail "npm not found: $NPM_BIN" +fi + +if [[ -z "$BRANCH" ]]; then + BRANCH="$(git -C "$WEB_ROOT" symbolic-ref --quiet --short HEAD || true)" +fi +[[ -n "$BRANCH" ]] || fail "cannot infer govoplan-web branch; pass --branch" + +CATALOG_PATH="$WEB_ROOT/public/catalogs/v1/channels/$CHANNEL.json" +KEYRING_PATH="$WEB_ROOT/public/catalogs/v1/keyring.json" +TAG_NAME="catalog-v$VERSION" + +run() { + printf '+' + printf ' %q' "$@" + printf '\n' + if [[ "$DRY_RUN" -eq 0 ]]; then + "$@" + fi +} + +GEN_ARGS=( + "$PYTHON" "$CORE_ROOT/scripts/generate-release-catalog.py" + --version "$VERSION" + --channel "$CHANNEL" + --expires-days "$EXPIRES_DAYS" + --catalog-output "$CATALOG_PATH" + --keyring-output "$KEYRING_PATH" + --public-base-url "$PUBLIC_BASE_URL" +) +if [[ -n "$SEQUENCE" ]]; then + GEN_ARGS+=(--sequence "$SEQUENCE") +fi +for signing_key in "${SIGNING_KEYS[@]}"; do + GEN_ARGS+=(--catalog-signing-key "$signing_key") +done + +run "${GEN_ARGS[@]}" + +if [[ "$DRY_RUN" -eq 0 ]]; then + GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE="$KEYRING_PATH" \ + "$PYTHON" -m govoplan_core.commands.module_installer \ + --validate-package-catalog "$CATALOG_PATH" \ + --require-signed-catalog \ + --approved-catalog-channel "$CHANNEL" \ + --format json >/dev/null +else + echo "+ GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE=$KEYRING_PATH $PYTHON -m govoplan_core.commands.module_installer --validate-package-catalog $CATALOG_PATH --require-signed-catalog --approved-catalog-channel $CHANNEL --format json" +fi + +if [[ "$BUILD_WEB" -eq 1 ]]; then + run env PATH="$(dirname "$NPM_BIN"):$PATH" "$NPM_BIN" --prefix "$WEB_ROOT" run build +fi + +if [[ "$COMMIT" -eq 1 ]]; then + run git -C "$WEB_ROOT" add "$CATALOG_PATH" "$KEYRING_PATH" + if [[ "$DRY_RUN" -eq 0 ]]; then + if git -C "$WEB_ROOT" diff --cached --quiet; then + echo "No govoplan-web catalog changes to commit." + else + run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog" + fi + else + run git -C "$WEB_ROOT" commit -m "Publish GovOPlaN v$VERSION $CHANNEL catalog" + fi +fi + +if [[ "$TAG_WEB" -eq 1 ]]; then + if git -C "$WEB_ROOT" rev-parse --quiet --verify "refs/tags/$TAG_NAME" >/dev/null; then + fail "govoplan-web tag already exists: $TAG_NAME" + fi + run git -C "$WEB_ROOT" tag -a "$TAG_NAME" -m "Publish GovOPlaN v$VERSION $CHANNEL catalog" +fi + +if [[ "$PUSH" -eq 1 ]]; then + if [[ "$TAG_WEB" -eq 1 ]]; then + run git -C "$WEB_ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/$BRANCH" "refs/tags/$TAG_NAME" + else + run git -C "$WEB_ROOT" push "$REMOTE" "HEAD:refs/heads/$BRANCH" + fi +fi + +echo "Catalog ready:" +echo " $CATALOG_PATH" +echo " $KEYRING_PATH" +echo " URL: $PUBLIC_BASE_URL/catalogs/v1/channels/$CHANNEL.json" diff --git a/scripts/push-release-tag.sh b/scripts/push-release-tag.sh index 3c65062..7e3a732 100644 --- a/scripts/push-release-tag.sh +++ b/scripts/push-release-tag.sh @@ -6,8 +6,9 @@ usage() { Usage: scripts/push-release-tag.sh [options] -Bumps all GovOPlaN package versions, commits all changes in all four repos, -creates an annotated release tag in each repo, and pushes branch+tag. +Bumps installable GovOPlaN package versions, commits pending changes in the +GovOPlaN release/module repositories, creates an annotated release tag in each +repo, and pushes branch+tag. By default the script asks which version part to bump: major X.Y.Z -> (X+1).0.0 @@ -23,16 +24,46 @@ Options: -m, --message Commit message. Defaults to "Release v". --tag-message Annotated tag message. Defaults to the commit message. --skip-release-lock Do not regenerate core webui/package-lock.release.json. + --publish-web-catalog After core/modules are pushed, publish a signed release + catalog into govoplan-web. + --web-root govoplan-web checkout for --publish-web-catalog. + Defaults to ../govoplan-web. + --catalog-signing-key + Catalog signing key for --publish-web-catalog. May be + repeated during key rotation. + --catalog-channel + Catalog channel. Defaults to stable. + --catalog-sequence Monotonic catalog sequence. Defaults to UTC timestamp. + --catalog-expires-days + Catalog expiry window. Defaults to 90. + --catalog-public-url + Public website URL. Defaults to + https://govoplan.add-ideas.de. + --build-web-catalog Run npm run build in govoplan-web before committing the + catalog update. --npm npm executable for lockfile generation. -y, --yes Do not prompt before committing, tagging, and pushing. -n, --dry-run Print intended actions without changing files or git state. -h, --help Show this help. Repos: - govoplan-core + Installable release packages: + govoplan-access + govoplan-admin + govoplan-tenancy + govoplan-policy + govoplan-audit govoplan-files govoplan-mail govoplan-campaign + govoplan-calendar + govoplan-core + + Roadmap/scaffold module repositories are tag-only until they have package + metadata: addresses, appointments, cases, connectors, dms, erp, + fit-connect, forms, identity-trust, idm, ledger, notifications, ops, + payments, portal, reporting, scheduling, search, tasks, templates, workflow, xoev, + xrechnung, and xta-osci. Examples: scripts/push-release-tag.sh @@ -48,10 +79,47 @@ fail() { ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PARENT="$(dirname "$ROOT")" -REPOS=( +PACKAGE_MODULE_REPOS=( + "$PARENT/govoplan-access" + "$PARENT/govoplan-admin" + "$PARENT/govoplan-tenancy" + "$PARENT/govoplan-policy" + "$PARENT/govoplan-audit" "$PARENT/govoplan-files" "$PARENT/govoplan-mail" "$PARENT/govoplan-campaign" + "$PARENT/govoplan-calendar" +) +TAG_ONLY_MODULE_REPOS=( + "$PARENT/govoplan-addresses" + "$PARENT/govoplan-appointments" + "$PARENT/govoplan-cases" + "$PARENT/govoplan-connectors" + "$PARENT/govoplan-dms" + "$PARENT/govoplan-erp" + "$PARENT/govoplan-fit-connect" + "$PARENT/govoplan-forms" + "$PARENT/govoplan-identity-trust" + "$PARENT/govoplan-idm" + "$PARENT/govoplan-ledger" + "$PARENT/govoplan-notifications" + "$PARENT/govoplan-ops" + "$PARENT/govoplan-payments" + "$PARENT/govoplan-portal" + "$PARENT/govoplan-reporting" + "$PARENT/govoplan-scheduling" + "$PARENT/govoplan-search" + "$PARENT/govoplan-tasks" + "$PARENT/govoplan-templates" + "$PARENT/govoplan-workflow" + "$PARENT/govoplan-xoev" + "$PARENT/govoplan-xrechnung" + "$PARENT/govoplan-xta-osci" +) +MODULE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "${TAG_ONLY_MODULE_REPOS[@]}") +PACKAGE_REPOS=("${PACKAGE_MODULE_REPOS[@]}" "$ROOT") +REPOS=( + "${MODULE_REPOS[@]}" "$ROOT" ) @@ -65,6 +133,14 @@ DRY_RUN=0 YES=0 GENERATE_RELEASE_LOCK=1 NPM_BIN="${NPM:-}" +PUBLISH_WEB_CATALOG=0 +WEB_ROOT="$PARENT/govoplan-web" +CATALOG_CHANNEL="stable" +CATALOG_SEQUENCE="" +CATALOG_EXPIRES_DAYS="90" +CATALOG_PUBLIC_URL="https://govoplan.add-ideas.de" +BUILD_WEB_CATALOG=0 +CATALOG_SIGNING_KEYS=() if [[ -z "${PYTHON:-}" ]]; then if [[ -x "$ROOT/.venv/bin/python" ]]; then @@ -175,8 +251,8 @@ if version_count != 1: if name != "govoplan-core": text, dependency_count = re.subn( - r'govoplan-core>=\d+\.\d+\.\d+', - f'govoplan-core>={new_version}', + r'(govoplan-[A-Za-z0-9-]+>=)\d+\.\d+\.\d+', + rf'\g<1>{new_version}', text, ) if dependency_count < 1: @@ -195,7 +271,22 @@ update_manifest_version() { case "$project_name" in govoplan-core) - manifest_path="$repo/src/govoplan_core/access/manifest.py" + return 0 + ;; + govoplan-access) + manifest_path="$repo/src/govoplan_access/backend/manifest.py" + ;; + govoplan-admin) + manifest_path="$repo/src/govoplan_admin/backend/manifest.py" + ;; + govoplan-tenancy) + manifest_path="$repo/src/govoplan_tenancy/backend/manifest.py" + ;; + govoplan-policy) + manifest_path="$repo/src/govoplan_policy/backend/manifest.py" + ;; + govoplan-audit) + manifest_path="$repo/src/govoplan_audit/backend/manifest.py" ;; govoplan-files) manifest_path="$repo/src/govoplan_files/backend/manifest.py" @@ -206,6 +297,9 @@ update_manifest_version() { govoplan-campaign) manifest_path="$repo/src/govoplan_campaign/backend/manifest.py" ;; + govoplan-calendar) + manifest_path="$repo/src/govoplan_calendar/backend/manifest.py" + ;; *) fail "unknown project for manifest version update: $project_name" ;; @@ -281,7 +375,7 @@ data = json.loads(path.read_text()) data["version"] = new_version dependencies = data.setdefault("dependencies", {}) for name, spec in list(dependencies.items()): - if name in {"@govoplan/files-webui", "@govoplan/mail-webui", "@govoplan/campaign-webui"}: + if name.startswith("@govoplan/") and isinstance(spec, str) and spec.startswith("git+"): dependencies[name] = re.sub(r"#v\d+\.\d+\.\d+$", f"#v{new_version}", spec) path.write_text(json.dumps(data, indent=2) + "\n") PYCODE @@ -420,6 +514,44 @@ while [[ $# -gt 0 ]]; do GENERATE_RELEASE_LOCK=0 shift ;; + --publish-web-catalog) + PUBLISH_WEB_CATALOG=1 + shift + ;; + --web-root) + [[ $# -ge 2 ]] || fail "missing value for $1" + WEB_ROOT="$2" + shift 2 + ;; + --catalog-signing-key) + [[ $# -ge 2 ]] || fail "missing value for $1" + CATALOG_SIGNING_KEYS+=("$2") + shift 2 + ;; + --catalog-channel) + [[ $# -ge 2 ]] || fail "missing value for $1" + CATALOG_CHANNEL="$2" + shift 2 + ;; + --catalog-sequence) + [[ $# -ge 2 ]] || fail "missing value for $1" + CATALOG_SEQUENCE="$2" + shift 2 + ;; + --catalog-expires-days) + [[ $# -ge 2 ]] || fail "missing value for $1" + CATALOG_EXPIRES_DAYS="$2" + shift 2 + ;; + --catalog-public-url) + [[ $# -ge 2 ]] || fail "missing value for $1" + CATALOG_PUBLIC_URL="$2" + shift 2 + ;; + --build-web-catalog) + BUILD_WEB_CATALOG=1 + shift + ;; --npm) [[ $# -ge 2 ]] || fail "missing value for $1" NPM_BIN="$2" @@ -448,6 +580,9 @@ done if [[ -n "$BUMP" && -n "$TARGET_VERSION" ]]; then fail "use either --bump or --version, not both" fi +if [[ "$PUBLISH_WEB_CATALOG" -eq 1 && ${#CATALOG_SIGNING_KEYS[@]} -eq 0 ]]; then + fail "--publish-web-catalog requires at least one --catalog-signing-key" +fi prompt_for_bump @@ -457,19 +592,30 @@ fi if ! command -v "$NPM_BIN" >/dev/null 2>&1; then fail "npm executable not found: $NPM_BIN" fi +if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then + [[ -d "$WEB_ROOT/.git" ]] || fail "govoplan-web repo not found: $WEB_ROOT" + [[ -x "$ROOT/scripts/publish-release-catalog.sh" ]] || fail "missing executable catalog publisher: $ROOT/scripts/publish-release-catalog.sh" +fi declare -A PROJECT_NAMES declare -A OLD_VERSIONS declare -A BRANCHES +declare -A REPO_KINDS for repo in "${REPOS[@]}"; do [[ -d "$repo/.git" ]] || fail "not a git repo: $repo" - [[ -f "$repo/pyproject.toml" ]] || fail "missing pyproject.toml: $repo" - PROJECT_NAMES["$repo"]="$(get_project_name "$repo")" - OLD_VERSIONS["$repo"]="$(get_project_version "$repo")" + if [[ -f "$repo/pyproject.toml" ]]; then + REPO_KINDS["$repo"]="package" + PROJECT_NAMES["$repo"]="$(get_project_name "$repo")" + OLD_VERSIONS["$repo"]="$(get_project_version "$repo")" - validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo" + validate_version "${OLD_VERSIONS[$repo]}" || fail "unsupported version ${OLD_VERSIONS[$repo]} in $repo" + else + REPO_KINDS["$repo"]="tag-only" + PROJECT_NAMES["$repo"]="$(basename "$repo")" + OLD_VERSIONS["$repo"]="tag-only" + fi if [[ -n "$BRANCH" ]]; then BRANCHES["$repo"]="$BRANCH" @@ -482,7 +628,7 @@ for repo in "${REPOS[@]}"; do git -C "$repo" remote get-url "$REMOTE" >/dev/null || fail "remote $REMOTE not found in $repo" upstream="$(git -C "$repo" rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)" - if [[ -n "$upstream" ]]; then + if [[ -n "$upstream" && "$upstream" != "@{u}" ]]; then behind_count="$(git -C "$repo" rev-list --count "HEAD..$upstream")" [[ "$behind_count" == "0" ]] || fail "$repo is behind $upstream by $behind_count commit(s); pull/rebase first" fi @@ -490,7 +636,7 @@ done if [[ -z "$TARGET_VERSION" ]]; then BASE_VERSION="${OLD_VERSIONS[$ROOT]}" - for repo in "${REPOS[@]}"; do + for repo in "${PACKAGE_REPOS[@]}"; do if [[ "${OLD_VERSIONS[$repo]}" != "$BASE_VERSION" ]]; then fail "repo versions differ; pass --version x.y.z to choose an explicit target" fi @@ -500,7 +646,7 @@ fi validate_version "$TARGET_VERSION" || fail "target version must be x.y.z: $TARGET_VERSION" -for repo in "${REPOS[@]}"; do +for repo in "${PACKAGE_REPOS[@]}"; do if [[ "$TARGET_VERSION" == "${OLD_VERSIONS[$repo]}" ]]; then continue fi @@ -537,22 +683,33 @@ echo " version: $TARGET_VERSION" echo " tag: $TAG" echo " remote: $REMOTE" echo " commit message: $COMMIT_MESSAGE" +echo " package repos: ${#PACKAGE_REPOS[@]} versioned" +echo " tag-only module repos: ${#TAG_ONLY_MODULE_REPOS[@]} committed/tagged/pushed without package version files" if [[ "$GENERATE_RELEASE_LOCK" -eq 1 ]]; then echo " release lock: regenerate core webui/package-lock.release.json after module tags are pushed" else echo " release lock: skipped" fi +if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then + echo " web catalog: publish $CATALOG_CHANNEL catalog to $WEB_ROOT after core tag is pushed" +else + echo " web catalog: skipped" +fi echo for repo in "${REPOS[@]}"; do - echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}" + if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then + echo "${PROJECT_NAMES[$repo]}: ${OLD_VERSIONS[$repo]} -> $TARGET_VERSION on ${BRANCHES[$repo]}" + else + echo "${PROJECT_NAMES[$repo]}: tag-only $TAG on ${BRANCHES[$repo]}" + fi git -C "$repo" status --short echo done confirm_release -for repo in "${REPOS[@]}"; do +for repo in "${PACKAGE_REPOS[@]}"; do if [[ "$DRY_RUN" -eq 1 ]]; then echo "Would update version files in $repo to $TARGET_VERSION" else @@ -562,20 +719,26 @@ done refresh_development_webui_lock -for repo in "${REPOS[@]:0:3}"; do +for repo in "${MODULE_REPOS[@]}"; do run git -C "$repo" add -A if [[ "$DRY_RUN" -eq 0 ]]; then if git -C "$repo" diff --cached --quiet; then - fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump" + if [[ "${REPO_KINDS[$repo]}" == "package" ]]; then + fail "no staged changes for ${PROJECT_NAMES[$repo]} after version bump" + fi + echo "No staged changes for ${PROJECT_NAMES[$repo]}; tagging current HEAD." + else + run git -C "$repo" commit -m "$COMMIT_MESSAGE" fi + else + run git -C "$repo" commit -m "$COMMIT_MESSAGE" fi - run git -C "$repo" commit -m "$COMMIT_MESSAGE" run git -C "$repo" tag -a "$TAG" -m "$TAG_MESSAGE" done -for repo in "${REPOS[@]:0:3}"; do +for repo in "${MODULE_REPOS[@]}"; do run git -C "$repo" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$repo]}" "refs/tags/$TAG" done @@ -593,6 +756,39 @@ run git -C "$ROOT" commit -m "$COMMIT_MESSAGE" run git -C "$ROOT" tag -a "$TAG" -m "$TAG_MESSAGE" run git -C "$ROOT" push --atomic "$REMOTE" "HEAD:refs/heads/${BRANCHES[$ROOT]}" "refs/tags/$TAG" +if [[ "$PUBLISH_WEB_CATALOG" -eq 1 ]]; then + CATALOG_ARGS=( + "$ROOT/scripts/publish-release-catalog.sh" + --version "$TARGET_VERSION" + --channel "$CATALOG_CHANNEL" + --expires-days "$CATALOG_EXPIRES_DAYS" + --core-root "$ROOT" + --web-root "$WEB_ROOT" + --public-base-url "$CATALOG_PUBLIC_URL" + --npm "$NPM_BIN" + --commit + --tag + --push + --remote "$REMOTE" + ) + if [[ -n "$BRANCH" ]]; then + CATALOG_ARGS+=(--branch "$BRANCH") + fi + if [[ -n "$CATALOG_SEQUENCE" ]]; then + CATALOG_ARGS+=(--sequence "$CATALOG_SEQUENCE") + fi + if [[ "$BUILD_WEB_CATALOG" -eq 1 ]]; then + CATALOG_ARGS+=(--build-web) + fi + for signing_key in "${CATALOG_SIGNING_KEYS[@]}"; do + CATALOG_ARGS+=(--catalog-signing-key "$signing_key") + done + if [[ "$DRY_RUN" -eq 1 ]]; then + CATALOG_ARGS+=(--dry-run) + fi + run "${CATALOG_ARGS[@]}" +fi + if [[ "$DRY_RUN" -eq 1 ]]; then echo "Dry run complete for $TAG across all GovOPlaN repos." else diff --git a/src/govoplan_core/access/__init__.py b/src/govoplan_core/access/__init__.py deleted file mode 100644 index 3c69ae1..0000000 --- a/src/govoplan_core/access/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Tenant, identity, auth, RBAC, and datastore access platform module.""" - diff --git a/src/govoplan_core/access/auth/__init__.py b/src/govoplan_core/access/auth/__init__.py deleted file mode 100644 index 3e58a06..0000000 --- a/src/govoplan_core/access/auth/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Authentication and principal helpers for platform access.""" - diff --git a/src/govoplan_core/access/auth/principals.py b/src/govoplan_core/access/auth/principals.py deleted file mode 100644 index eec1834..0000000 --- a/src/govoplan_core/access/auth/principals.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -from govoplan_core.access.permissions.evaluator import scopes_grant - - -AuthMethod = Literal["session", "api_key", "service_account"] - - -@dataclass(frozen=True, slots=True) -class Principal: - account_id: str - membership_id: str | None - tenant_id: str | None - scopes: frozenset[str] - group_ids: frozenset[str] - auth_method: AuthMethod - api_key_id: str | None = None - session_id: str | None = None - service_account_id: str | None = None - email: str | None = None - display_name: str | None = None - - def has(self, required: str) -> bool: - return scopes_grant(self.scopes, required) - diff --git a/src/govoplan_core/access/auth/roles.py b/src/govoplan_core/access/auth/roles.py deleted file mode 100644 index 1d66f1f..0000000 --- a/src/govoplan_core/access/auth/roles.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from sqlalchemy.orm import Session - -from govoplan_core.access.db.models import Account, Group, GroupMembership, Membership, Role, RoleBinding -from govoplan_core.access.permissions.evaluator import expand_scopes -from govoplan_core.core.modules import PermissionDefinition, SubjectType - - -def membership_group_ids(session: Session, *, tenant_id: str, membership_id: str) -> frozenset[str]: - rows = ( - session.query(Group.id) - .join(GroupMembership, GroupMembership.group_id == Group.id) - .filter( - GroupMembership.tenant_id == tenant_id, - GroupMembership.membership_id == membership_id, - Group.is_active.is_(True), - ) - .all() - ) - return frozenset(row[0] for row in rows) - - -def roles_for_subject( - session: Session, - *, - subject_type: SubjectType, - subject_id: str, - tenant_id: str | None, -) -> list[Role]: - query = ( - session.query(Role) - .join(RoleBinding, RoleBinding.role_id == Role.id) - .filter( - RoleBinding.subject_type == subject_type, - RoleBinding.subject_id == subject_id, - ) - ) - if tenant_id is None: - query = query.filter(RoleBinding.tenant_id.is_(None), Role.tenant_id.is_(None)) - else: - query = query.filter(RoleBinding.tenant_id == tenant_id, Role.tenant_id == tenant_id) - return query.order_by(Role.name.asc()).all() - - -def membership_roles(session: Session, membership: Membership) -> list[Role]: - roles_by_id = { - role.id: role - for role in roles_for_subject( - session, - subject_type="membership", - subject_id=membership.id, - tenant_id=membership.tenant_id, - ) - } - for group_id in membership_group_ids(session, tenant_id=membership.tenant_id, membership_id=membership.id): - for role in roles_for_subject(session, subject_type="group", subject_id=group_id, tenant_id=membership.tenant_id): - roles_by_id[role.id] = role - return list(roles_by_id.values()) - - -def account_system_roles(session: Session, account: Account) -> list[Role]: - return roles_for_subject(session, subject_type="account", subject_id=account.id, tenant_id=None) - - -def principal_scopes( - session: Session, - *, - account: Account, - membership: Membership | None, - include_system: bool, - catalog: dict[str, PermissionDefinition], -) -> list[str]: - scopes: set[str] = set() - if membership is not None: - for role in membership_roles(session, membership): - scopes.update(role.permissions or []) - if include_system: - for role in account_system_roles(session, account): - scopes.update(role.permissions or []) - return expand_scopes(scopes, catalog=catalog) - diff --git a/src/govoplan_core/access/auth/tokens.py b/src/govoplan_core/access/auth/tokens.py deleted file mode 100644 index a1bf8e1..0000000 --- a/src/govoplan_core/access/auth/tokens.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac -import secrets - - -DEFAULT_RANDOM_BYTES = 32 - - -def generate_secret(prefix: str, *, random_bytes: int = DEFAULT_RANDOM_BYTES) -> str: - return prefix + secrets.token_urlsafe(random_bytes) - - -def hash_secret(secret: str) -> str: - return hashlib.sha256(secret.encode("utf-8")).hexdigest() - - -def verify_secret(secret: str, expected_hash: str) -> bool: - return hmac.compare_digest(hash_secret(secret), expected_hash) - diff --git a/src/govoplan_core/access/db/__init__.py b/src/govoplan_core/access/db/__init__.py deleted file mode 100644 index 81aa90c..0000000 --- a/src/govoplan_core/access/db/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Access-platform SQLAlchemy metadata and models.""" - diff --git a/src/govoplan_core/access/db/base.py b/src/govoplan_core/access/db/base.py deleted file mode 100644 index 2bfbd5f..0000000 --- a/src/govoplan_core/access/db/base.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from sqlalchemy import JSON, MetaData -from sqlalchemy.orm import DeclarativeBase - -from govoplan_core.db.base import NAMING_CONVENTION, TimestampMixin, utcnow - - -class AccessBase(DeclarativeBase): - metadata = MetaData(naming_convention=NAMING_CONVENTION) - - type_annotation_map = { - dict[str, Any]: JSON, - list[dict[str, Any]]: JSON, - list[str]: JSON, - } - - -__all__ = ["AccessBase", "NAMING_CONVENTION", "TimestampMixin", "utcnow"] diff --git a/src/govoplan_core/access/db/models.py b/src/govoplan_core/access/db/models.py deleted file mode 100644 index f77ed8b..0000000 --- a/src/govoplan_core/access/db/models.py +++ /dev/null @@ -1,237 +0,0 @@ -from __future__ import annotations - -import uuid -from datetime import datetime -from typing import Any - -from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, text -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from govoplan_core.access.db.base import AccessBase, TimestampMixin - - -def new_uuid() -> str: - return str(uuid.uuid4()) - - -class Account(AccessBase, TimestampMixin): - __tablename__ = "access_accounts" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - email: Mapped[str] = mapped_column(String(320), nullable=False) - normalized_email: Mapped[str] = mapped_column(String(320), nullable=False, unique=True, index=True) - display_name: Mapped[str | None] = mapped_column(String(255)) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - auth_provider: Mapped[str] = mapped_column(String(50), default="local", nullable=False) - password_hash: Mapped[str | None] = mapped_column(String(500)) - password_reset_required: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - - memberships: Mapped[list[Membership]] = relationship(back_populates="account", cascade="all, delete-orphan") - - -class Tenant(AccessBase, TimestampMixin): - __tablename__ = "access_tenants" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - slug: Mapped[str] = mapped_column(String(100), nullable=False, unique=True, index=True) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - - memberships: Mapped[list[Membership]] = relationship(back_populates="tenant", cascade="all, delete-orphan") - - -class Membership(AccessBase, TimestampMixin): - __tablename__ = "access_memberships" - __table_args__ = (UniqueConstraint("tenant_id", "account_id", name="uq_access_memberships_tenant_account"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=False, index=True) - account_id: Mapped[str] = mapped_column(ForeignKey("access_accounts.id", ondelete="CASCADE"), nullable=False, index=True) - email_snapshot: Mapped[str | None] = mapped_column(String(320), index=True) - display_name: Mapped[str | None] = mapped_column(String(255)) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - account: Mapped[Account] = relationship(back_populates="memberships") - tenant: Mapped[Tenant] = relationship(back_populates="memberships") - - -class Group(AccessBase, TimestampMixin): - __tablename__ = "access_groups" - __table_args__ = (UniqueConstraint("tenant_id", "slug", name="uq_access_groups_tenant_slug"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=False, index=True) - slug: Mapped[str] = mapped_column(String(100), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - template_id: Mapped[str | None] = mapped_column(String(100), index=True) - is_protected: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - -class GroupMembership(AccessBase, TimestampMixin): - __tablename__ = "access_group_memberships" - __table_args__ = ( - UniqueConstraint("tenant_id", "membership_id", "group_id", name="uq_access_group_memberships_member_group"), - ) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=False, index=True) - membership_id: Mapped[str] = mapped_column(ForeignKey("access_memberships.id", ondelete="CASCADE"), nullable=False, index=True) - group_id: Mapped[str] = mapped_column(ForeignKey("access_groups.id", ondelete="CASCADE"), nullable=False, index=True) - - -class Role(AccessBase, TimestampMixin): - __tablename__ = "access_roles" - __table_args__ = ( - UniqueConstraint("tenant_id", "slug", name="uq_access_roles_tenant_slug"), - Index( - "uq_access_roles_system_slug", - "slug", - unique=True, - sqlite_where=text("tenant_id IS NULL"), - postgresql_where=text("tenant_id IS NULL"), - ), - ) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str | None] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=True, index=True) - slug: Mapped[str] = mapped_column(String(100), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - permissions: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) - level: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True) - is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - is_assignable: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - template_id: Mapped[str | None] = mapped_column(String(100), index=True) - is_protected: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - - -class RoleBinding(AccessBase, TimestampMixin): - __tablename__ = "access_role_bindings" - __table_args__ = ( - Index("ix_access_role_bindings_subject", "subject_type", "subject_id"), - Index("ix_access_role_bindings_tenant_subject", "tenant_id", "subject_type", "subject_id"), - ) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str | None] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=True, index=True) - role_id: Mapped[str] = mapped_column(ForeignKey("access_roles.id", ondelete="CASCADE"), nullable=False, index=True) - subject_type: Mapped[str] = mapped_column(String(50), nullable=False) - subject_id: Mapped[str] = mapped_column(String(36), nullable=False) - created_by_account_id: Mapped[str | None] = mapped_column(String(36), index=True) - created_by_membership_id: Mapped[str | None] = mapped_column(String(36), index=True) - - -class PermissionCatalogEntry(AccessBase, TimestampMixin): - __tablename__ = "access_permission_catalog" - - scope: Mapped[str] = mapped_column(String(255), primary_key=True) - module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True) - resource: Mapped[str] = mapped_column(String(100), nullable=False) - action: Mapped[str] = mapped_column(String(100), nullable=False) - label: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str] = mapped_column(Text, default="", nullable=False) - category: Mapped[str] = mapped_column(String(100), nullable=False) - level: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - is_deprecated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - - -class RoleTemplateModel(AccessBase, TimestampMixin): - __tablename__ = "access_role_templates" - __table_args__ = (UniqueConstraint("module_id", "level", "slug", name="uq_access_role_templates_module_level_slug"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True) - slug: Mapped[str] = mapped_column(String(100), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - permissions: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) - level: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False) - managed: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - protected: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - - -class ModuleInstallation(AccessBase, TimestampMixin): - __tablename__ = "access_module_installations" - - module_id: Mapped[str] = mapped_column(String(100), primary_key=True) - version: Mapped[str] = mapped_column(String(50), nullable=False) - enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - -class AuthSession(AccessBase, TimestampMixin): - __tablename__ = "access_auth_sessions" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str | None] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=True, index=True) - membership_id: Mapped[str | None] = mapped_column(ForeignKey("access_memberships.id", ondelete="CASCADE"), nullable=True, index=True) - account_id: Mapped[str] = mapped_column(ForeignKey("access_accounts.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True) - csrf_token_hash: Mapped[str | None] = mapped_column(String(255)) - expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True) - last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) - user_agent: Mapped[str | None] = mapped_column(String(500)) - ip_address: Mapped[str | None] = mapped_column(String(100)) - - -class ApiKey(AccessBase, TimestampMixin): - __tablename__ = "access_api_keys" - __table_args__ = (Index("ix_access_api_keys_owner", "owner_subject_type", "owner_subject_id"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=False, index=True) - owner_subject_type: Mapped[str] = mapped_column(String(50), nullable=False) - owner_subject_id: Mapped[str] = mapped_column(String(36), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - prefix: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - key_hash: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) - scopes: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) - - -class TenantDatastore(AccessBase, TimestampMixin): - __tablename__ = "access_tenant_datastores" - __table_args__ = (UniqueConstraint("tenant_id", "module_id", name="uq_access_tenant_datastores_tenant_module"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), nullable=False, index=True) - module_id: Mapped[str] = mapped_column(String(100), default="*", nullable=False) - mode: Mapped[str] = mapped_column(String(20), default="shared", nullable=False) - database_url_secret_ref: Mapped[str | None] = mapped_column(String(255)) - schema_name: Mapped[str | None] = mapped_column(String(100)) - storage_bucket: Mapped[str | None] = mapped_column(String(255)) - region: Mapped[str | None] = mapped_column(String(100)) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - - -class SystemSettings(AccessBase, TimestampMixin): - __tablename__ = "access_system_settings" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default="global") - default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False) - allow_tenant_custom_groups: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - allow_tenant_custom_roles: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - allow_tenant_api_keys: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - -class TenantSettings(AccessBase, TimestampMixin): - __tablename__ = "access_tenant_settings" - - tenant_id: Mapped[str] = mapped_column(ForeignKey("access_tenants.id", ondelete="CASCADE"), primary_key=True) - allow_custom_groups: Mapped[bool | None] = mapped_column(Boolean) - allow_custom_roles: Mapped[bool | None] = mapped_column(Boolean) - allow_api_keys: Mapped[bool | None] = mapped_column(Boolean) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - diff --git a/src/govoplan_core/access/db/session.py b/src/govoplan_core/access/db/session.py deleted file mode 100644 index 4977466..0000000 --- a/src/govoplan_core/access/db/session.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from collections.abc import Generator - -from sqlalchemy.engine import Engine -from sqlalchemy.orm import Session, sessionmaker - -from govoplan_core.db.session import create_database_engine - - -def create_access_engine(database_url: str) -> Engine: - return create_database_engine(database_url) - - -def create_access_session_factory(database_url: str) -> sessionmaker[Session]: - engine = create_access_engine(database_url) - return sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False) - - -def session_scope(session_factory: sessionmaker[Session]) -> Generator[Session, None, None]: - with session_factory() as session: - yield session diff --git a/src/govoplan_core/access/manifest.py b/src/govoplan_core/access/manifest.py deleted file mode 100644 index cd71561..0000000 --- a/src/govoplan_core/access/manifest.py +++ /dev/null @@ -1,118 +0,0 @@ -from __future__ import annotations - -from govoplan_core.access.db.base import AccessBase -from govoplan_core.access.db import models as access_models # noqa: F401 - populate access metadata -from govoplan_core.core.modules import MigrationSpec, ModuleManifest, PermissionDefinition, RoleTemplate - - -def _permission(scope: str, label: str, description: str, category: str, level: str) -> PermissionDefinition: - module_id, resource, action = scope.split(":", 2) - return PermissionDefinition( - scope=scope, - label=label, - description=description, - category=category, - level=level, # type: ignore[arg-type] - module_id=module_id, - resource=resource, - action=action, - ) - - -ACCESS_PERMISSIONS: tuple[PermissionDefinition, ...] = ( - _permission("access:tenant:read", "View tenants", "List and inspect tenant registry entries.", "Access", "system"), - _permission("access:tenant:create", "Create tenants", "Create tenant registry entries.", "Access", "system"), - _permission("access:tenant:update", "Update tenants", "Update tenant metadata and activation state.", "Access", "system"), - _permission("access:account:read", "View accounts", "List and inspect global login accounts.", "Access", "system"), - _permission("access:account:create", "Create accounts", "Create global login accounts.", "Access", "system"), - _permission("access:account:update", "Update accounts", "Update or suspend global login accounts.", "Access", "system"), - _permission("access:membership:read", "View memberships", "List tenant memberships and effective access.", "Tenant access", "tenant"), - _permission("access:membership:create", "Create memberships", "Create tenant-local account memberships.", "Tenant access", "tenant"), - _permission("access:membership:update", "Update memberships", "Update or suspend tenant memberships.", "Tenant access", "tenant"), - _permission("access:group:read", "View groups", "List tenant groups and members.", "Tenant access", "tenant"), - _permission("access:group:write", "Manage groups", "Create and update tenant groups.", "Tenant access", "tenant"), - _permission("access:group:manage_members", "Manage group members", "Add and remove memberships from groups.", "Tenant access", "tenant"), - _permission("access:role:read", "View roles", "Inspect tenant and system role definitions.", "Tenant access", "tenant"), - _permission("access:role:write", "Manage roles", "Create and update assignable roles.", "Tenant access", "tenant"), - _permission("access:role:assign", "Assign roles", "Bind roles to memberships, groups, accounts or services.", "Tenant access", "tenant"), - _permission("access:api_key:read", "View API keys", "List API keys without revealing secrets.", "Tenant access", "tenant"), - _permission("access:api_key:create", "Create API keys", "Create tenant API keys within delegation limits.", "Tenant access", "tenant"), - _permission("access:api_key:revoke", "Revoke API keys", "Revoke tenant API keys.", "Tenant access", "tenant"), - _permission("access:setting:read", "View settings", "Read access and governance settings.", "Tenant access", "tenant"), - _permission("access:setting:write", "Manage settings", "Update access and governance settings.", "Tenant access", "tenant"), - _permission("access:governance:read", "View governance", "Inspect managed role and group templates.", "Access", "system"), - _permission("access:governance:write", "Manage governance", "Create and assign managed role and group templates.", "Access", "system"), -) - -ACCESS_ROLE_TEMPLATES: tuple[RoleTemplate, ...] = ( - RoleTemplate( - slug="system_owner", - name="System owner", - description="Protected full instance-wide administration.", - permissions=("system:*",), - level="system", - managed=True, - protected=True, - ), - RoleTemplate( - slug="system_admin", - name="System administrator", - description="Manage tenants, accounts, settings, and governance without protected owner status.", - permissions=( - "access:tenant:read", - "access:tenant:create", - "access:tenant:update", - "access:account:read", - "access:account:create", - "access:account:update", - "access:governance:read", - "access:governance:write", - ), - level="system", - managed=True, - protected=False, - ), - RoleTemplate( - slug="tenant_owner", - name="Tenant owner", - description="Protected full tenant administration and module access.", - permissions=("tenant:*",), - level="tenant", - managed=True, - protected=True, - ), - RoleTemplate( - slug="access_admin", - name="Access administrator", - description="Manage memberships, groups, roles, and API keys within delegation limits.", - permissions=( - "access:membership:read", - "access:membership:create", - "access:membership:update", - "access:group:read", - "access:group:write", - "access:group:manage_members", - "access:role:read", - "access:role:assign", - "access:api_key:read", - "access:api_key:create", - "access:api_key:revoke", - ), - level="tenant", - managed=True, - protected=False, - ), -) - -manifest = ModuleManifest( - id="access", - name="Access", - version="0.1.4", - permissions=ACCESS_PERMISSIONS, - role_templates=ACCESS_ROLE_TEMPLATES, - migration_spec=MigrationSpec(module_id="access", metadata=AccessBase.metadata), -) - - -def get_manifest() -> ModuleManifest: - return manifest diff --git a/src/govoplan_core/access/permissions/__init__.py b/src/govoplan_core/access/permissions/__init__.py deleted file mode 100644 index 74957d6..0000000 --- a/src/govoplan_core/access/permissions/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Permission definitions, evaluation, and registry helpers.""" - diff --git a/src/govoplan_core/access/permissions/definitions.py b/src/govoplan_core/access/permissions/definitions.py deleted file mode 100644 index f3ebd78..0000000 --- a/src/govoplan_core/access/permissions/definitions.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import annotations - -from govoplan_core.core.modules import PermissionDefinition, PermissionLevel, RoleTemplate - -__all__ = ["PermissionDefinition", "PermissionLevel", "RoleTemplate"] - diff --git a/src/govoplan_core/access/permissions/evaluator.py b/src/govoplan_core/access/permissions/evaluator.py deleted file mode 100644 index 823f997..0000000 --- a/src/govoplan_core/access/permissions/evaluator.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterable, Mapping - -from govoplan_core.core.modules import PermissionDefinition - - -def scope_grants(granted: str, required: str, *, catalog: Mapping[str, PermissionDefinition] | None = None) -> bool: - if granted == required or granted == "*": - return True - if granted == "tenant:*": - if catalog is None: - return not required.startswith("system:") - definition = catalog.get(required) - return definition is not None and definition.level == "tenant" - if granted == "system:*": - if catalog is None: - return required.startswith("system:") - definition = catalog.get(required) - return definition is not None and definition.level == "system" - if granted.endswith(":*"): - return required.startswith(granted[:-1]) - return False - - -def scopes_grant( - scopes: Iterable[str], - required: str, - *, - catalog: Mapping[str, PermissionDefinition] | None = None, -) -> bool: - return any(scope_grants(scope, required, catalog=catalog) for scope in scopes) - - -def expand_scopes( - scopes: Iterable[str], - *, - catalog: Mapping[str, PermissionDefinition], - include_unknown: bool = False, -) -> list[str]: - expanded: set[str] = set() - for scope in {str(scope) for scope in scopes if scope}: - matched = {candidate for candidate in catalog if scope_grants(scope, candidate, catalog=catalog)} - if matched: - expanded.update(matched) - if scope.endswith(":*") or scope in {"*", "tenant:*", "system:*"}: - expanded.add(scope) - continue - if include_unknown: - expanded.add(scope) - return sorted(expanded) - diff --git a/src/govoplan_core/access/permissions/registry.py b/src/govoplan_core/access/permissions/registry.py deleted file mode 100644 index f9efad0..0000000 --- a/src/govoplan_core/access/permissions/registry.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterable - -from govoplan_core.access.permissions.evaluator import expand_scopes, scopes_grant -from govoplan_core.core.modules import PermissionDefinition - - -class PermissionRegistry: - def __init__(self, permissions: Iterable[PermissionDefinition] = ()) -> None: - self._catalog: dict[str, PermissionDefinition] = {} - for permission in permissions: - self.register(permission) - - @property - def catalog(self) -> dict[str, PermissionDefinition]: - return dict(self._catalog) - - def register(self, permission: PermissionDefinition) -> None: - if permission.scope in self._catalog: - raise ValueError(f"Duplicate permission scope: {permission.scope}") - self._catalog[permission.scope] = permission - - def has(self, scope: str) -> bool: - return scope in self._catalog - - def grants(self, scopes: Iterable[str], required: str) -> bool: - return scopes_grant(scopes, required, catalog=self._catalog) - - def expand(self, scopes: Iterable[str], *, include_unknown: bool = False) -> list[str]: - return expand_scopes(scopes, catalog=self._catalog, include_unknown=include_unknown) - - def validate_known(self, scopes: Iterable[str]) -> list[str]: - unknown = sorted(scope for scope in scopes if scope not in self._catalog and not scope.endswith(":*") and scope not in {"*", "tenant:*", "system:*"}) - if unknown: - raise ValueError("Unknown permission scope(s): " + ", ".join(unknown)) - return sorted(set(scopes)) - diff --git a/src/govoplan_core/access/tenancy/__init__.py b/src/govoplan_core/access/tenancy/__init__.py deleted file mode 100644 index abea063..0000000 --- a/src/govoplan_core/access/tenancy/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Tenant datastore routing abstractions.""" - diff --git a/src/govoplan_core/access/tenancy/datastore.py b/src/govoplan_core/access/tenancy/datastore.py deleted file mode 100644 index 28550f5..0000000 --- a/src/govoplan_core/access/tenancy/datastore.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from collections.abc import Generator -from contextlib import contextmanager -from dataclasses import dataclass -from typing import Literal, Protocol - -from sqlalchemy.orm import Session, sessionmaker - - -DatastoreMode = Literal["shared", "schema", "database"] - - -@dataclass(frozen=True, slots=True) -class TenantDatastoreRef: - tenant_id: str - mode: DatastoreMode = "shared" - datastore_id: str | None = None - schema_name: str | None = None - database_url_secret_ref: str | None = None - - -class TenantDatastore(Protocol): - ref: TenantDatastoreRef - - @contextmanager - def session_for(self, module_id: str) -> Generator[Session, None, None]: - ... - - -class DatastoreResolver(Protocol): - def for_tenant(self, tenant_id: str) -> TenantDatastore: - ... - - -class SharedTenantDatastore: - def __init__(self, ref: TenantDatastoreRef, session_factory: sessionmaker[Session]) -> None: - self.ref = ref - self._session_factory = session_factory - - @contextmanager - def session_for(self, module_id: str) -> Generator[Session, None, None]: - del module_id - with self._session_factory() as session: - yield session - - -class SharedDatastoreResolver: - def __init__(self, session_factory: sessionmaker[Session]) -> None: - self._session_factory = session_factory - - def for_tenant(self, tenant_id: str) -> TenantDatastore: - return SharedTenantDatastore(TenantDatastoreRef(tenant_id=tenant_id), self._session_factory) - diff --git a/src/govoplan_core/admin/common.py b/src/govoplan_core/admin/common.py new file mode 100644 index 0000000..d5a4349 --- /dev/null +++ b/src/govoplan_core/admin/common.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import re + +_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +class AdminConflictError(ValueError): + pass + + +class AdminValidationError(ValueError): + pass + + +def slugify(value: str) -> str: + slug = _SLUG_RE.sub("-", value.strip().casefold()).strip("-") + if not slug: + raise AdminValidationError("A slug must contain at least one letter or number.") + return slug[:100] + diff --git a/src/govoplan_core/admin/governance.py b/src/govoplan_core/admin/governance.py deleted file mode 100644 index 0d91bb0..0000000 --- a/src/govoplan_core/admin/governance.py +++ /dev/null @@ -1,328 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from sqlalchemy.orm import Session - -from govoplan_core.admin.service import AdminConflictError, AdminValidationError, slugify -from govoplan_core.core.optional import reraise_unless_missing_package -from govoplan_core.db.models import ( - ApiKey, - GovernanceTemplate, - GovernanceTemplateAssignment, - Group, - GroupRoleAssignment, - Role, - SystemSettings, - Tenant, - UserGroupMembership, - UserRoleAssignment, -) -from govoplan_core.security.permissions import validate_tenant_permissions - -SYSTEM_SETTINGS_ID = "global" -TEMPLATE_KINDS = {"group", "role"} -ASSIGNMENT_MODES = {"available", "required"} - - -def _file_models(): - try: - from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare - except ModuleNotFoundError as exc: - reraise_unless_missing_package(exc, "govoplan_files") - return None - return FileAsset, FileFolder, FileShare - - -@dataclass(frozen=True) -class EffectiveTenantGovernance: - allow_custom_groups: bool - allow_custom_roles: bool - allow_api_keys: bool - - -def get_system_settings(session: Session) -> SystemSettings: - item = session.get(SystemSettings, SYSTEM_SETTINGS_ID) - if item is None: - item = SystemSettings(id=SYSTEM_SETTINGS_ID) - session.add(item) - session.flush() - return item - - -def _narrowing_bool(system_allows: bool, tenant_override: bool | None) -> bool: - if not system_allows: - return False - return tenant_override is not False - - -def effective_tenant_governance(session: Session, tenant: Tenant) -> EffectiveTenantGovernance: - defaults = get_system_settings(session) - return EffectiveTenantGovernance( - allow_custom_groups=_narrowing_bool(defaults.allow_tenant_custom_groups, tenant.allow_custom_groups), - allow_custom_roles=_narrowing_bool(defaults.allow_tenant_custom_roles, tenant.allow_custom_roles), - allow_api_keys=_narrowing_bool(defaults.allow_tenant_api_keys, tenant.allow_api_keys), - ) - - -def assert_tenant_governance_override_allowed(session: Session, *, field: str, value: bool | None) -> None: - if value is not True: - return - defaults = get_system_settings(session) - blocked = { - "allow_custom_groups": not defaults.allow_tenant_custom_groups, - "allow_custom_roles": not defaults.allow_tenant_custom_roles, - "allow_api_keys": not defaults.allow_tenant_api_keys, - } - if blocked.get(field): - raise AdminValidationError("Tenant governance cannot explicitly allow a capability denied by system settings.") - - -def validate_template(kind: str, permissions: list[str]) -> list[str]: - if kind not in TEMPLATE_KINDS: - raise AdminValidationError("Template kind must be group or role.") - if kind == "group": - if permissions: - raise AdminValidationError("Group templates do not contain permissions; assign roles to groups inside each tenant.") - return [] - try: - return validate_tenant_permissions(permissions) - except ValueError as exc: - raise AdminValidationError(str(exc)) from exc - - -def create_template( - session: Session, - *, - kind: str, - slug: str, - name: str, - description: str | None, - permissions: list[str], - is_active: bool, - assignments: list[dict[str, str]], -) -> GovernanceTemplate: - normalized_slug = slugify(slug) - permissions = validate_template(kind, permissions) - exists = session.query(GovernanceTemplate).filter( - GovernanceTemplate.kind == kind, - GovernanceTemplate.slug == normalized_slug, - ).first() - if exists: - raise AdminConflictError(f"A {kind} template with slug {normalized_slug!r} already exists.") - item = GovernanceTemplate( - kind=kind, - slug=normalized_slug, - name=name.strip(), - description=description, - permissions=permissions, - is_active=is_active, - ) - session.add(item) - session.flush() - set_template_assignments(session, item, assignments) - return item - - -def update_template( - session: Session, - item: GovernanceTemplate, - *, - name: str, - description: str | None, - permissions: list[str], - is_active: bool, - assignments: list[dict[str, str]], -) -> GovernanceTemplate: - item.name = name.strip() - item.description = description - item.permissions = validate_template(item.kind, permissions) - item.is_active = is_active - set_template_assignments(session, item, assignments) - sync_template(session, item) - return item - - -def set_template_assignments( - session: Session, - item: GovernanceTemplate, - assignments: list[dict[str, str]], -) -> None: - desired: dict[str, str] = {} - for assignment in assignments: - tenant_id = assignment.get("tenant_id", "") - mode = assignment.get("mode", "available") - if mode not in ASSIGNMENT_MODES: - raise AdminValidationError("Template assignment mode must be available or required.") - tenant = session.get(Tenant, tenant_id) - if tenant is None: - raise AdminValidationError(f"Unknown tenant: {tenant_id}") - desired[tenant_id] = mode - - existing = { - row.tenant_id: row - for row in session.query(GovernanceTemplateAssignment) - .filter(GovernanceTemplateAssignment.template_id == item.id) - .all() - } - for tenant_id, row in list(existing.items()): - if tenant_id in desired: - row.mode = desired[tenant_id] - continue - _remove_materialized_template(session, item, tenant_id) - session.delete(row) - - for tenant_id, mode in desired.items(): - if tenant_id not in existing: - session.add(GovernanceTemplateAssignment(template_id=item.id, tenant_id=tenant_id, mode=mode)) - session.flush() - sync_template(session, item) - - -def sync_template(session: Session, item: GovernanceTemplate) -> None: - assignments = session.query(GovernanceTemplateAssignment).filter( - GovernanceTemplateAssignment.template_id == item.id - ).all() - for assignment in assignments: - required = assignment.mode == "required" - if item.kind == "group": - group = session.query(Group).filter( - Group.tenant_id == assignment.tenant_id, - Group.system_template_id == item.id, - ).first() - if group is None: - group = Group( - tenant_id=assignment.tenant_id, - slug=_available_slug(session, Group, assignment.tenant_id, item.slug), - name=item.name, - description=item.description, - is_active=item.is_active, - system_template_id=item.id, - system_required=required, - ) - session.add(group) - else: - group.name = item.name - group.description = item.description - group.system_required = required - if required: - group.is_active = item.is_active - else: - role = session.query(Role).filter( - Role.tenant_id == assignment.tenant_id, - Role.system_template_id == item.id, - ).first() - if role is None: - role = Role( - tenant_id=assignment.tenant_id, - slug=_available_slug(session, Role, assignment.tenant_id, item.slug), - name=item.name, - description=item.description, - permissions=item.permissions, - is_builtin=False, - is_assignable=item.is_active, - system_template_id=item.id, - system_required=required, - ) - session.add(role) - else: - role.name = item.name - role.description = item.description - role.permissions = item.permissions - role.system_required = required - if required: - role.is_assignable = item.is_active - session.flush() - - - -def delete_template(session: Session, item: GovernanceTemplate) -> None: - assignments = session.query(GovernanceTemplateAssignment).filter( - GovernanceTemplateAssignment.template_id == item.id - ).all() - for assignment in assignments: - _remove_materialized_template(session, item, assignment.tenant_id) - session.delete(item) - - -def _remove_materialized_template(session: Session, item: GovernanceTemplate, tenant_id: str) -> None: - if item.kind == "group": - group = session.query(Group).filter( - Group.tenant_id == tenant_id, - Group.system_template_id == item.id, - ).first() - if group is None: - return - membership_count = session.query(UserGroupMembership).filter(UserGroupMembership.group_id == group.id).count() - role_count = session.query(GroupRoleAssignment).filter(GroupRoleAssignment.group_id == group.id).count() - file_models = _file_models() - owned_asset_count = 0 - owned_folder_count = 0 - shared_asset_count = 0 - if file_models is not None: - FileAsset, FileFolder, FileShare = file_models - owned_asset_count = session.query(FileAsset).filter(FileAsset.owner_group_id == group.id).count() - owned_folder_count = session.query(FileFolder).filter(FileFolder.owner_group_id == group.id).count() - shared_asset_count = session.query(FileShare).filter( - FileShare.target_type == "group", FileShare.target_id == group.id - ).count() - if membership_count or role_count or owned_asset_count or owned_folder_count or shared_asset_count: - raise AdminConflictError( - f"Cannot remove {item.name!r} from the tenant while its managed group has members, roles, files, folders or shares." - ) - session.delete(group) - return - - role = session.query(Role).filter( - Role.tenant_id == tenant_id, - Role.system_template_id == item.id, - ).first() - if role is None: - return - user_count = session.query(UserRoleAssignment).filter(UserRoleAssignment.role_id == role.id).count() - group_count = session.query(GroupRoleAssignment).filter(GroupRoleAssignment.role_id == role.id).count() - if user_count or group_count: - raise AdminConflictError( - f"Cannot remove {item.name!r} from the tenant while its managed role is assigned to users or groups." - ) - session.delete(role) - - -def _available_slug(session: Session, model: type[Group] | type[Role], tenant_id: str, base: str) -> str: - candidate = base - suffix = 2 - while session.query(model).filter(model.tenant_id == tenant_id, model.slug == candidate).first(): - candidate = f"{base}-{suffix}" - suffix += 1 - return candidate - - -def ensure_group_mutation_allowed(group: Group, *, requested_active: bool | None = None) -> None: - if group.system_template_id and group.system_required and requested_active is False: - raise AdminConflictError("This centrally required group cannot be deactivated by the tenant.") - - -def ensure_role_mutation_allowed(role: Role, *, requested_assignable: bool | None = None) -> None: - if role.system_template_id: - if role.system_required and requested_assignable is False: - raise AdminConflictError("This centrally required role cannot be made unavailable by the tenant.") - raise AdminConflictError("Centrally managed role definitions can only be changed in System administration.") - - -def assert_api_keys_allowed(session: Session, tenant: Tenant) -> None: - if not effective_tenant_governance(session, tenant).allow_api_keys: - raise AdminConflictError("API-key creation is disabled by system or tenant governance.") - - -def assert_custom_groups_allowed(session: Session, tenant: Tenant) -> None: - if not effective_tenant_governance(session, tenant).allow_custom_groups: - raise AdminConflictError("Custom tenant groups are disabled by system or tenant governance.") - - -def assert_custom_roles_allowed(session: Session, tenant: Tenant) -> None: - if not effective_tenant_governance(session, tenant).allow_custom_roles: - raise AdminConflictError("Custom tenant roles are disabled by system or tenant governance.") - - -def active_api_key_count(session: Session, tenant_id: str) -> int: - return session.query(ApiKey).filter(ApiKey.tenant_id == tenant_id, ApiKey.revoked_at.is_(None)).count() diff --git a/src/govoplan_core/admin/models.py b/src/govoplan_core/admin/models.py new file mode 100644 index 0000000..96297f7 --- /dev/null +++ b/src/govoplan_core/admin/models.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy import Boolean, JSON, String +from sqlalchemy.orm import Mapped, mapped_column + +from govoplan_core.db.base import Base, TimestampMixin + + +class SystemSettings(Base, TimestampMixin): + __tablename__ = "system_settings" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default="global") + default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False) + allow_tenant_custom_groups: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + allow_tenant_custom_roles: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + allow_tenant_api_keys: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + + +__all__ = ["SystemSettings"] + diff --git a/src/govoplan_core/admin/service.py b/src/govoplan_core/admin/service.py deleted file mode 100644 index cf6da50..0000000 --- a/src/govoplan_core/admin/service.py +++ /dev/null @@ -1,609 +0,0 @@ -from __future__ import annotations - -import re -import secrets -import string -from collections.abc import Iterable -from dataclasses import dataclass - -from sqlalchemy import func -from sqlalchemy.orm import Session - -from govoplan_core.core.optional import reraise_unless_missing_package -from govoplan_core.db.models import ( - Account, - ApiKey, - Group, - GroupRoleAssignment, - Role, - SystemRoleAssignment, - Tenant, - User, - UserGroupMembership, - UserRoleAssignment, -) -from govoplan_core.security.passwords import hash_password -from govoplan_core.security.permissions import ( - DEFAULT_SYSTEM_ROLES, - DEFAULT_TENANT_ROLES, - normalize_email, - scopes_grant, - validate_system_permissions, - validate_tenant_permissions, -) - -_SLUG_RE = re.compile(r"[^a-z0-9]+") -_TEMP_PASSWORD_ALPHABET = string.ascii_letters + string.digits + "-_!@#" - - -class AdminConflictError(ValueError): - pass - - - - -class AdminValidationError(ValueError): - pass - - -@dataclass(slots=True) -class MembershipCreationResult: - user: User - account: Account - temporary_password: str | None = None - account_created: bool = False - - -def slugify(value: str) -> str: - slug = _SLUG_RE.sub("-", value.strip().casefold()).strip("-") - if not slug: - raise AdminValidationError("A slug must contain at least one letter or number.") - return slug[:100] - - -def generate_temporary_password(length: int = 20) -> str: - return "".join(secrets.choice(_TEMP_PASSWORD_ALPHABET) for _ in range(length)) - - -def ensure_default_roles(session: Session, tenant: Tenant | None = None) -> dict[str, Role]: - definitions = DEFAULT_TENANT_ROLES if tenant is not None else DEFAULT_SYSTEM_ROLES - roles: dict[str, Role] = {} - for slug, definition in definitions.items(): - query = session.query(Role).filter(Role.slug == slug) - query = query.filter(Role.tenant_id == tenant.id) if tenant is not None else query.filter(Role.tenant_id.is_(None)) - role = query.one_or_none() - if role is None: - role = Role( - tenant_id=tenant.id if tenant is not None else None, - slug=slug, - name=str(definition["name"]), - description=str(definition.get("description") or "") or None, - permissions=list(definition["permissions"]), - is_builtin=bool(definition.get("is_builtin", True)), - is_assignable=bool(definition.get("is_assignable", True)), - ) - session.add(role) - session.flush() - else: - # Tenant built-ins and explicitly managed system roles remain - # code-defined. Seeded, non-protected system roles are only created - # here and can subsequently be administered in the System roles UI. - managed = tenant is not None or bool(definition.get("managed", False)) - if managed: - role.name = str(definition["name"]) - role.description = str(definition.get("description") or "") or None - role.permissions = list(definition["permissions"]) - role.is_builtin = bool(definition.get("is_builtin", True)) - role.is_assignable = bool(definition.get("is_assignable", True)) - session.add(role) - roles[slug] = role - return roles - - -def get_or_create_account( - session: Session, - *, - email: str, - display_name: str | None = None, - password: str | None = None, - password_reset_required: bool = False, -) -> tuple[Account, bool, str | None]: - normalized = normalize_email(email) - if not normalized or "@" not in normalized: - raise AdminValidationError("Enter a valid email address.") - account = session.query(Account).filter(Account.normalized_email == normalized).one_or_none() - if account is not None: - if not account.is_active: - raise AdminConflictError( - "The global account is disabled. A system administrator must reactivate it before adding a tenant membership." - ) - return account, False, None - - temporary_password = password or generate_temporary_password() - account = Account( - email=email.strip(), - normalized_email=normalized, - display_name=display_name.strip() if display_name else None, - is_active=True, - auth_provider="local", - password_hash=hash_password(temporary_password), - password_reset_required=password_reset_required or password is None, - ) - session.add(account) - session.flush() - return account, True, temporary_password - - -def create_membership( - session: Session, - *, - tenant: Tenant, - email: str, - display_name: str | None = None, - password: str | None = None, - password_reset_required: bool = False, - is_active: bool = True, -) -> MembershipCreationResult: - account, account_created, temporary_password = get_or_create_account( - session, - email=email, - display_name=display_name, - password=password, - password_reset_required=password_reset_required, - ) - existing = ( - session.query(User) - .filter(User.tenant_id == tenant.id, User.account_id == account.id) - .one_or_none() - ) - if existing is not None: - raise AdminConflictError("This account already belongs to the tenant.") - - user = User( - tenant_id=tenant.id, - account_id=account.id, - email=account.email, - display_name=display_name.strip() if display_name else account.display_name, - is_active=is_active, - is_tenant_admin=False, - auth_provider=account.auth_provider, - password_hash=account.password_hash, - ) - session.add(user) - session.flush() - return MembershipCreationResult( - user=user, - account=account, - temporary_password=temporary_password if account_created else None, - account_created=account_created, - ) - - - -def set_user_groups(session: Session, *, user: User, group_ids: Iterable[str]) -> None: - ids = sorted(set(group_ids)) - groups = ( - session.query(Group) - .filter(Group.tenant_id == user.tenant_id, Group.id.in_(ids)) - .all() - if ids else [] - ) - if len(groups) != len(ids): - raise AdminValidationError("One or more selected groups do not belong to the tenant.") - - session.query(UserGroupMembership).filter( - UserGroupMembership.tenant_id == user.tenant_id, - UserGroupMembership.user_id == user.id, - ).delete(synchronize_session=False) - for group in groups: - session.add(UserGroupMembership(tenant_id=user.tenant_id, user_id=user.id, group_id=group.id)) - session.flush() - - -def set_user_roles(session: Session, *, user: User, role_ids: Iterable[str]) -> None: - ids = sorted(set(role_ids)) - roles = ( - session.query(Role) - .filter(Role.tenant_id == user.tenant_id, Role.id.in_(ids), Role.is_assignable.is_(True)) - .all() - if ids else [] - ) - if len(roles) != len(ids): - raise AdminValidationError("One or more selected roles are invalid or not assignable.") - - session.query(UserRoleAssignment).filter( - UserRoleAssignment.tenant_id == user.tenant_id, - UserRoleAssignment.user_id == user.id, - ).delete(synchronize_session=False) - for role in roles: - session.add(UserRoleAssignment(tenant_id=user.tenant_id, user_id=user.id, role_id=role.id)) - session.flush() - - -def set_group_members(session: Session, *, group: Group, user_ids: Iterable[str]) -> None: - ids = sorted(set(user_ids)) - users = ( - session.query(User) - .filter(User.tenant_id == group.tenant_id, User.id.in_(ids)) - .all() - if ids else [] - ) - if len(users) != len(ids): - raise AdminValidationError("One or more selected users do not belong to the tenant.") - - session.query(UserGroupMembership).filter( - UserGroupMembership.tenant_id == group.tenant_id, - UserGroupMembership.group_id == group.id, - ).delete(synchronize_session=False) - for user in users: - session.add(UserGroupMembership(tenant_id=group.tenant_id, user_id=user.id, group_id=group.id)) - session.flush() - - -def set_group_roles(session: Session, *, group: Group, role_ids: Iterable[str]) -> None: - ids = sorted(set(role_ids)) - roles = ( - session.query(Role) - .filter(Role.tenant_id == group.tenant_id, Role.id.in_(ids), Role.is_assignable.is_(True)) - .all() - if ids else [] - ) - if len(roles) != len(ids): - raise AdminValidationError("One or more selected roles are invalid or not assignable.") - - session.query(GroupRoleAssignment).filter( - GroupRoleAssignment.tenant_id == group.tenant_id, - GroupRoleAssignment.group_id == group.id, - ).delete(synchronize_session=False) - for role in roles: - session.add(GroupRoleAssignment(tenant_id=group.tenant_id, group_id=group.id, role_id=role.id)) - session.flush() - - -def create_custom_role( - session: Session, - *, - tenant: Tenant, - slug: str, - name: str, - description: str | None, - permissions: Iterable[str], -) -> Role: - normalized_slug = slugify(slug) - if session.query(Role).filter(Role.tenant_id == tenant.id, Role.slug == normalized_slug).count(): - raise AdminConflictError("A role with this slug already exists in the tenant.") - try: - validated = validate_tenant_permissions(permissions) - except ValueError as exc: - raise AdminValidationError(str(exc)) from exc - role = Role( - tenant_id=tenant.id, - slug=normalized_slug, - name=name.strip(), - description=description.strip() if description else None, - permissions=validated, - is_builtin=False, - is_assignable=True, - ) - session.add(role) - session.flush() - return role - - -def update_custom_role( - session: Session, - *, - role: Role, - name: str, - description: str | None, - permissions: Iterable[str], - is_assignable: bool, -) -> None: - if role.is_builtin: - raise AdminConflictError("Built-in roles are managed by the application and cannot be edited.") - try: - validated = validate_tenant_permissions(permissions) - except ValueError as exc: - raise AdminValidationError(str(exc)) from exc - role.name = name.strip() - role.description = description.strip() if description else None - role.permissions = validated - role.is_assignable = is_assignable - session.add(role) - session.flush() - - -def delete_custom_role(session: Session, role: Role) -> None: - if role.is_builtin: - raise AdminConflictError("Built-in roles cannot be deleted.") - assigned = session.query(UserRoleAssignment).filter(UserRoleAssignment.role_id == role.id).count() - assigned += session.query(GroupRoleAssignment).filter(GroupRoleAssignment.role_id == role.id).count() - if assigned: - raise AdminConflictError("Remove all user and group assignments before deleting this role.") - session.delete(role) - session.flush() - - -def create_system_role( - session: Session, - *, - slug: str, - name: str, - description: str | None, - permissions: Iterable[str], -) -> Role: - normalized_slug = slugify(slug) - if session.query(Role).filter(Role.tenant_id.is_(None), Role.slug == normalized_slug).count(): - raise AdminConflictError("A system role with this slug already exists.") - try: - validated = validate_system_permissions(permissions) - except ValueError as exc: - raise AdminValidationError(str(exc)) from exc - if "system:*" in validated: - raise AdminValidationError("Only the protected System owner role may contain system:*.") - role = Role( - tenant_id=None, - slug=normalized_slug, - name=name.strip(), - description=description.strip() if description else None, - permissions=validated, - is_builtin=False, - is_assignable=True, - ) - session.add(role) - session.flush() - return role - - -def update_system_role( - session: Session, - *, - role: Role, - name: str, - description: str | None, - permissions: Iterable[str], - is_assignable: bool, -) -> None: - if role.tenant_id is not None: - raise AdminValidationError("This is not a system role.") - if role.slug == "system_owner": - raise AdminConflictError("The protected System owner role cannot be edited.") - try: - validated = validate_system_permissions(permissions) - except ValueError as exc: - raise AdminValidationError(str(exc)) from exc - if "system:*" in validated: - raise AdminValidationError("Only the protected System owner role may contain system:*.") - role.name = name.strip() - role.description = description.strip() if description else None - role.permissions = validated - role.is_assignable = is_assignable - role.is_builtin = False - session.add(role) - session.flush() - - -def delete_system_role(session: Session, role: Role) -> None: - if role.tenant_id is not None: - raise AdminValidationError("This is not a system role.") - if role.slug == "system_owner": - raise AdminConflictError("The protected System owner role cannot be deleted.") - assigned = session.query(SystemRoleAssignment).filter(SystemRoleAssignment.role_id == role.id).count() - if assigned: - raise AdminConflictError("Remove all account assignments before deleting this system role.") - session.delete(role) - session.flush() - - -def assert_can_delegate_system_permissions(actor_scopes: Iterable[str], permissions: Iterable[str]) -> None: - """Prevent a system administrator from defining stronger roles than they hold.""" - from govoplan_core.security.permissions import delegateable_system_scopes - - requested = set(validate_system_permissions(permissions)) - if "system:*" in requested: - if not scopes_grant(actor_scopes, "system:*"): - raise AdminValidationError("Only a System owner may delegate full system access.") - return - allowed = delegateable_system_scopes(actor_scopes) - missing = sorted(scope for scope in requested if scope not in allowed) - if missing: - raise AdminValidationError("You cannot delegate system permissions you do not currently hold: " + ", ".join(missing)) - - -def assert_can_delegate_system_roles( - session: Session, - *, - actor_scopes: Iterable[str], - role_ids: Iterable[str], -) -> None: - ids = sorted(set(role_ids)) - if not ids: - return - roles = session.query(Role).filter(Role.tenant_id.is_(None), Role.id.in_(ids)).all() - if len(roles) != len(ids): - raise AdminValidationError("One or more selected system roles are invalid.") - for role in roles: - assert_can_delegate_system_permissions(actor_scopes, role.permissions or []) - - -def set_system_roles(session: Session, *, account: Account, role_ids: Iterable[str]) -> None: - ids = sorted(set(role_ids)) - roles = ( - session.query(Role) - .filter(Role.tenant_id.is_(None), Role.id.in_(ids), Role.is_assignable.is_(True)) - .all() - if ids else [] - ) - if len(roles) != len(ids): - raise AdminValidationError("One or more system roles are invalid or not assignable.") - for role in roles: - try: - validate_system_permissions(role.permissions or []) - except ValueError as exc: - raise AdminValidationError(str(exc)) from exc - - session.query(SystemRoleAssignment).filter(SystemRoleAssignment.account_id == account.id).delete(synchronize_session=False) - for role in roles: - session.add(SystemRoleAssignment(account_id=account.id, role_id=role.id)) - session.flush() - assert_system_owner_exists(session) - - -def account_has_system_scope(session: Session, account: Account, required: str) -> bool: - roles = ( - session.query(Role) - .join(SystemRoleAssignment, SystemRoleAssignment.role_id == Role.id) - .filter(SystemRoleAssignment.account_id == account.id, Role.tenant_id.is_(None)) - .all() - ) - return any(scopes_grant(role.permissions or [], required) for role in roles) - - -def tenant_owner_user_ids(session: Session, tenant_id: str) -> set[str]: - """Return active tenant memberships that currently satisfy the operational-owner invariant.""" - - users = ( - session.query(User) - .join(Account, Account.id == User.account_id) - .filter( - User.tenant_id == tenant_id, - User.is_active.is_(True), - Account.is_active.is_(True), - ) - .all() - ) - owner_ids: set[str] = set() - user_ids = [user.id for user in users] - if not user_ids: - return owner_ids - - roles_by_user_id: dict[str, list[Role]] = {user_id: [] for user_id in user_ids} - direct_role_rows = ( - session.query(UserRoleAssignment.user_id, Role) - .join(Role, UserRoleAssignment.role_id == Role.id) - .filter( - UserRoleAssignment.tenant_id == tenant_id, - UserRoleAssignment.user_id.in_(user_ids), - Role.tenant_id == tenant_id, - ) - .all() - ) - for user_id, role in direct_role_rows: - roles_by_user_id.setdefault(user_id, []).append(role) - - group_role_rows = ( - session.query(UserGroupMembership.user_id, Role) - .join(GroupRoleAssignment, UserGroupMembership.group_id == GroupRoleAssignment.group_id) - .join(Role, GroupRoleAssignment.role_id == Role.id) - .join(Group, Group.id == UserGroupMembership.group_id) - .filter( - UserGroupMembership.tenant_id == tenant_id, - UserGroupMembership.user_id.in_(user_ids), - Group.is_active.is_(True), - Role.tenant_id == tenant_id, - ) - .all() - ) - for user_id, role in group_role_rows: - roles_by_user_id.setdefault(user_id, []).append(role) - - for user in users: - effective = [ - scope - for role in roles_by_user_id.get(user.id, []) - for scope in (role.permissions or []) - ] - if scopes_grant(effective, "admin:roles:write") and scopes_grant(effective, "campaign:send"): - owner_ids.add(user.id) - return owner_ids - - -def tenant_has_owner(session: Session, tenant_id: str) -> bool: - return bool(tenant_owner_user_ids(session, tenant_id)) - - -def assert_tenant_owner_exists(session: Session, tenant_id: str) -> None: - session.flush() - tenant = session.get(Tenant, tenant_id) - if tenant is not None and tenant.is_active and not tenant_has_owner(session, tenant_id): - raise AdminConflictError("An active tenant must retain at least one active owner or administrator with full operational access.") - - -def assert_system_owner_exists(session: Session) -> None: - """Keep one active assignment of the protected System owner role. - - Other roles may hold broad system permissions, but they are intentionally - not substitutes for the protected break-glass owner identity. - """ - session.flush() - owner_role = ( - session.query(Role) - .filter(Role.tenant_id.is_(None), Role.slug == "system_owner") - .one_or_none() - ) - if owner_role is None: - raise AdminConflictError("The protected System owner role is missing.") - count = ( - session.query(func.count(SystemRoleAssignment.id)) - .join(Account, Account.id == SystemRoleAssignment.account_id) - .filter(SystemRoleAssignment.role_id == owner_role.id, Account.is_active.is_(True)) - .scalar() - ) - if not count: - raise AdminConflictError("At least one active account must retain the protected System owner role.") - - -def role_assignment_counts(session: Session, role_id: str) -> tuple[int, int]: - return ( - session.query(UserRoleAssignment).filter(UserRoleAssignment.role_id == role_id).count(), - session.query(GroupRoleAssignment).filter(GroupRoleAssignment.role_id == role_id).count(), - ) - - -def _optional_model(module_name: str, model_name: str): - try: - module = __import__(module_name, fromlist=[model_name]) - except ModuleNotFoundError as exc: - reraise_unless_missing_package(exc, module_name.split(".", 1)[0]) - return None - return getattr(module, model_name) - - -def tenant_counts(session: Session, tenant_id: str) -> dict[str, int]: - Campaign = _optional_model("govoplan_campaign.backend.db.models", "Campaign") - FileAsset = _optional_model("govoplan_files.backend.db.models", "FileAsset") - - return { - "users": session.query(User).filter(User.tenant_id == tenant_id).count(), - "active_users": session.query(User).filter(User.tenant_id == tenant_id, User.is_active.is_(True)).count(), - "groups": session.query(Group).filter(Group.tenant_id == tenant_id).count(), - "campaigns": session.query(Campaign).filter(Campaign.tenant_id == tenant_id).count() if Campaign is not None else 0, - "files": session.query(FileAsset).filter(FileAsset.tenant_id == tenant_id).count() if FileAsset is not None else 0, - "api_keys": session.query(ApiKey).filter(ApiKey.tenant_id == tenant_id, ApiKey.revoked_at.is_(None)).count(), - } - - -def assert_can_delegate_tenant_permissions(actor_scopes: Iterable[str], permissions: Iterable[str]) -> None: - """Prevent administrators from defining or assigning roles beyond their own effective tenant powers.""" - from govoplan_core.security.permissions import delegateable_tenant_scopes - requested = set(validate_tenant_permissions(permissions)) - if "tenant:*" in requested: - # Only a tenant owner-equivalent can delegate the wildcard. - if not scopes_grant(actor_scopes, "tenant:*"): - raise AdminValidationError("You cannot delegate full tenant access unless you already have it.") - return - allowed = delegateable_tenant_scopes(actor_scopes) - missing = sorted(scope for scope in requested if scope not in allowed) - if missing: - raise AdminValidationError("You cannot delegate permissions you do not currently hold: " + ", ".join(missing)) - - -def assert_can_delegate_roles(session: Session, *, actor_scopes: Iterable[str], role_ids: Iterable[str], tenant_id: str) -> None: - ids = sorted(set(role_ids)) - if not ids: - return - roles = session.query(Role).filter(Role.tenant_id == tenant_id, Role.id.in_(ids)).all() - if len(roles) != len(ids): - raise AdminValidationError("One or more selected roles do not belong to the tenant.") - for role in roles: - assert_can_delegate_tenant_permissions(actor_scopes, role.permissions or []) diff --git a/src/govoplan_core/admin/settings.py b/src/govoplan_core/admin/settings.py new file mode 100644 index 0000000..acfe5df --- /dev/null +++ b/src/govoplan_core/admin/settings.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sqlalchemy.orm import Session + +from govoplan_core.admin.models import SystemSettings + +SYSTEM_SETTINGS_ID = "global" + + +def get_system_settings(session: Session) -> SystemSettings: + item = session.get(SystemSettings, SYSTEM_SETTINGS_ID) + if item is None: + item = SystemSettings(id=SYSTEM_SETTINGS_ID) + session.add(item) + session.flush() + return item diff --git a/src/govoplan_core/api/v1/admin.py b/src/govoplan_core/api/v1/admin.py deleted file mode 100644 index 0768186..0000000 --- a/src/govoplan_core/api/v1/admin.py +++ /dev/null @@ -1,1715 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone - -from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlalchemy import func -from sqlalchemy.orm import Session - -from govoplan_core.admin.governance import ( - assert_api_keys_allowed, - assert_custom_groups_allowed, - assert_custom_roles_allowed, - assert_tenant_governance_override_allowed, - create_template, - delete_template, - effective_tenant_governance, - ensure_group_mutation_allowed, - ensure_role_mutation_allowed, - get_system_settings, - update_template, -) -from govoplan_core.admin.service import ( - AdminConflictError, - AdminValidationError, - assert_can_delegate_roles, - assert_can_delegate_system_permissions, - assert_can_delegate_system_roles, - assert_can_delegate_tenant_permissions, - assert_system_owner_exists, - assert_tenant_owner_exists, - create_custom_role, - create_membership, - create_system_role, - delete_custom_role, - delete_system_role, - ensure_default_roles, - get_or_create_account, - role_assignment_counts, - set_group_members, - set_group_roles, - set_system_roles, - set_user_groups, - set_user_roles, - slugify, - tenant_counts, - tenant_owner_user_ids, - update_custom_role, - update_system_role, -) -from govoplan_core.api.v1.admin_schemas import ( - AdminApiKeyCreateRequest, - AdminApiKeyCreateResponse, - AdminOverviewResponse, - ApiKeyAdminItem, - ApiKeyListResponse, - AuditAdminItem, - AuditAdminListResponse, - GroupCreateRequest, - GroupListResponse, - GroupSummary, - GovernanceTemplateCreateRequest, - GovernanceTemplateItem, - GovernanceTemplateListResponse, - GovernanceTemplateUpdateRequest, - GroupUpdateRequest, - PermissionCatalogResponse, - PermissionItem, - PrivacyRetentionPolicyItem, - PrivacyRetentionPolicyScopeRequest, - PrivacyRetentionPolicyScopeResponse, - RetentionRunRequest, - RetentionRunResponse, - RoleCreateRequest, - RoleListResponse, - RoleSummary, - RoleUpdateRequest, - SystemAccountCreateRequest, - SystemAccountCreateResponse, - SystemAccountItem, - SystemAccountListResponse, - SystemAccountMembershipsUpdateRequest, - SystemAccountRolesUpdateRequest, - SystemAccountUpdateRequest, - SystemSettingsItem, - SystemSettingsUpdateRequest, - TenantAdminItem, - TenantCreateRequest, - TenantListResponse, - TenantOwnerCandidate, - TenantOwnerCandidateListResponse, - TenantSettingsItem, - TenantSettingsUpdateRequest, - TenantUpdateRequest, - UserAdminItem, - UserCreateRequest, - UserCreateResponse, - UserListResponse, - UserUpdateRequest, -) -from govoplan_core.auth.dependencies import ApiPrincipal, get_api_principal, has_scope, require_any_scope, require_scope -from govoplan_core.audit.logging import audit_event, audit_from_principal -from govoplan_core.db.models import ( - Account, - ApiKey, - AuditLog, - Group, - GovernanceTemplate, - GovernanceTemplateAssignment, - GroupRoleAssignment, - Role, - SystemRoleAssignment, - Tenant, - User, - UserGroupMembership, - UserRoleAssignment, -) -from govoplan_core.db.session import get_session -from govoplan_core.privacy.retention import ( - PrivacyPolicyError, - apply_retention_policy, - effective_privacy_policy, - effective_privacy_policy_sources, - get_privacy_policy_for_scope, - parent_privacy_policy, - parent_privacy_policy_sources, - privacy_policy_from_settings, - set_privacy_policy, - set_privacy_policy_for_scope, -) -from govoplan_core.security.api_keys import create_api_key -from govoplan_core.security.permissions import ALL_PERMISSIONS, effective_permission_count, normalize_email, scopes_grant -from govoplan_core.security.sessions import collect_direct_user_roles, collect_system_roles, collect_user_groups, collect_user_roles, collect_user_scopes -from govoplan_core.security.time import utc_now - -router = APIRouter(prefix="/admin", tags=["admin"]) - - -def _http_admin_error(exc: Exception) -> HTTPException: - if isinstance(exc, AdminConflictError): - return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) - if isinstance(exc, AdminValidationError): - return HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) - return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) - - -def _require_system_role_assignment(principal: ApiPrincipal) -> None: - if not (has_scope(principal, "system:roles:assign") or has_scope(principal, "system:access:assign")): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:roles:assign") - - -def _require_permission(principal: ApiPrincipal, scope: str) -> None: - if not has_scope(principal, scope): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Missing scope: {scope}") - - -def _resolve_tenant( - session: Session, - principal: ApiPrincipal, - tenant_id: str | None, -) -> Tenant: - target_id = tenant_id or principal.tenant_id - if target_id != principal.tenant_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Switch to the target tenant before using tenant-administration endpoints.", - ) - tenant = session.get(Tenant, target_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found") - return tenant - - -def _role_summary(session: Session, role: Role) -> RoleSummary: - group_count = 0 - if role.tenant_id is None: - user_count = session.query(SystemRoleAssignment).filter(SystemRoleAssignment.role_id == role.id).count() - permission_level = "system" - else: - user_count, group_count = role_assignment_counts(session, role.id) - permission_level = "tenant" - permissions = list(role.permissions or []) - return RoleSummary( - id=role.id, - slug=role.slug, - name=role.name, - description=role.description, - permissions=permissions, - effective_permission_count=effective_permission_count(permissions, level=permission_level), - is_builtin=role.is_builtin, - is_assignable=role.is_assignable, - user_assignments=user_count, - group_assignments=group_count, - level="system" if role.tenant_id is None else "tenant", - system_template_id=role.system_template_id, - system_required=role.system_required, - ) - - -def _group_summary(session: Session, group: Group, *, include_members: bool = True) -> GroupSummary: - member_ids = [ - row[0] - for row in session.query(UserGroupMembership.user_id) - .filter(UserGroupMembership.tenant_id == group.tenant_id, UserGroupMembership.group_id == group.id) - .all() - ] - roles = ( - session.query(Role) - .join(GroupRoleAssignment, GroupRoleAssignment.role_id == Role.id) - .filter(GroupRoleAssignment.tenant_id == group.tenant_id, GroupRoleAssignment.group_id == group.id) - .order_by(Role.name.asc()) - .all() - ) - return GroupSummary( - id=group.id, - slug=group.slug, - name=group.name, - description=group.description, - is_active=group.is_active, - member_count=len(member_ids), - member_ids=member_ids if include_members else [], - roles=[_role_summary(session, role) for role in roles], - created_at=group.created_at, - updated_at=group.updated_at, - system_template_id=group.system_template_id, - system_required=group.system_required, - ) - - -def _user_item(session: Session, user: User, *, owner_ids: set[str] | None = None) -> UserAdminItem: - account = session.get(Account, user.account_id) - if account is None: - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="User account is missing") - groups = collect_user_groups(session, user) - roles = collect_direct_user_roles(session, user) - effective_owner_ids = owner_ids if owner_ids is not None else tenant_owner_user_ids(session, user.tenant_id) - return UserAdminItem( - id=user.id, - account_id=account.id, - tenant_id=user.tenant_id, - email=account.email, - display_name=user.display_name or account.display_name, - is_active=user.is_active, - account_is_active=account.is_active, - password_reset_required=account.password_reset_required, - last_login_at=account.last_login_at, - groups=[_group_summary(session, group, include_members=False) for group in groups], - roles=[_role_summary(session, role) for role in roles], - effective_scopes=collect_user_scopes(session, user, include_system=False), - is_owner=user.id in effective_owner_ids, - is_last_active_owner=user.id in effective_owner_ids and len(effective_owner_ids) == 1, - created_at=user.created_at, - updated_at=user.updated_at, - ) - - -def _tenant_item(session: Session, tenant: Tenant) -> TenantAdminItem: - governance = effective_tenant_governance(session, tenant) - return TenantAdminItem( - id=tenant.id, - slug=tenant.slug, - name=tenant.name, - description=tenant.description, - default_locale=tenant.default_locale, - settings=tenant.settings or {}, - allow_custom_groups=tenant.allow_custom_groups, - allow_custom_roles=tenant.allow_custom_roles, - allow_api_keys=tenant.allow_api_keys, - effective_governance={ - "allow_custom_groups": governance.allow_custom_groups, - "allow_custom_roles": governance.allow_custom_roles, - "allow_api_keys": governance.allow_api_keys, - }, - is_active=tenant.is_active, - counts=tenant_counts(session, tenant.id), - created_at=tenant.created_at, - updated_at=tenant.updated_at, - ) - - -def _tenant_settings_item(tenant: Tenant) -> TenantSettingsItem: - return TenantSettingsItem( - id=tenant.id, - slug=tenant.slug, - name=tenant.name, - default_locale=tenant.default_locale, - settings=tenant.settings or {}, - ) - - -def _system_account_item(session: Session, account: Account) -> SystemAccountItem: - memberships = ( - session.query(User, Tenant) - .join(Tenant, Tenant.id == User.tenant_id) - .filter(User.account_id == account.id) - .order_by(Tenant.name.asc()) - .all() - ) - owner_ids_by_tenant = { - tenant.id: tenant_owner_user_ids(session, tenant.id) - for _, tenant in memberships - } - return SystemAccountItem( - account_id=account.id, - email=account.email, - display_name=account.display_name, - is_active=account.is_active, - memberships=[ - { - "tenant_id": tenant.id, - "tenant_name": tenant.name, - "user_id": user.id, - "is_active": user.is_active and tenant.is_active, - "role_ids": [role.id for role in collect_direct_user_roles(session, user)], - "group_ids": [group.id for group in collect_user_groups(session, user)], - "is_owner": user.id in owner_ids_by_tenant[tenant.id], - "is_last_active_owner": ( - user.id in owner_ids_by_tenant[tenant.id] - and len(owner_ids_by_tenant[tenant.id]) == 1 - ), - } - for user, tenant in memberships - ], - roles=[_role_summary(session, role) for role in collect_system_roles(session, account)], - last_login_at=account.last_login_at, - ) - - -def _api_key_item(session: Session, item: ApiKey) -> ApiKeyAdminItem: - user = session.get(User, item.user_id) - account = session.get(Account, user.account_id) if user else None - return ApiKeyAdminItem( - id=item.id, - user_id=item.user_id, - user_email=account.email if account else "Unknown account", - name=item.name, - prefix=item.prefix, - scopes=item.scopes or [], - expires_at=item.expires_at, - last_used_at=item.last_used_at, - revoked_at=item.revoked_at, - created_at=item.created_at, - ) - - -def _governance_template_item(session: Session, item: GovernanceTemplate) -> GovernanceTemplateItem: - assignments = session.query(GovernanceTemplateAssignment).filter( - GovernanceTemplateAssignment.template_id == item.id - ).order_by(GovernanceTemplateAssignment.tenant_id.asc()).all() - return GovernanceTemplateItem( - id=item.id, - kind=item.kind, - slug=item.slug, - name=item.name, - description=item.description, - permissions=item.permissions or [], - effective_permission_count=effective_permission_count(item.permissions or [], level="tenant") if item.kind == "role" else 0, - is_active=item.is_active, - assignments=[{"tenant_id": row.tenant_id, "mode": row.mode} for row in assignments], - created_at=item.created_at, - updated_at=item.updated_at, - ) - - -@router.get("/overview", response_model=AdminOverviewResponse) -def admin_overview( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_any_scope( - "admin:users:read", "admin:groups:read", "admin:roles:read", "admin:settings:read", - "admin:api_keys:read", "system:tenants:read", "system:accounts:read", "system:roles:read", "system:access:read", - "system:settings:read", "system:governance:read", "system:audit:read", - )), -): - tenant = session.get(Tenant, principal.tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found") - counts = tenant_counts(session, tenant.id) - capabilities = [item.scope for item in ALL_PERMISSIONS if has_scope(principal, item.scope)] - return AdminOverviewResponse( - active_tenant_id=tenant.id, - active_tenant_name=tenant.name, - tenant_count=session.query(Tenant).count() if has_scope(principal, "system:tenants:read") else None, - system_account_count=session.query(Account).count() if has_scope(principal, "system:accounts:read") or has_scope(principal, "system:access:read") else None, - system_group_template_count=session.query(GovernanceTemplate).filter(GovernanceTemplate.kind == "group").count() if has_scope(principal, "system:governance:read") else None, - system_role_template_count=session.query(GovernanceTemplate).filter(GovernanceTemplate.kind == "role").count() if has_scope(principal, "system:governance:read") else None, - user_count=counts["users"], - active_user_count=counts["active_users"], - group_count=counts["groups"], - role_count=session.query(Role).filter(Role.tenant_id == tenant.id).count(), - active_api_key_count=counts["api_keys"], - capabilities=capabilities, - ) - - -@router.get("/permissions", response_model=PermissionCatalogResponse) -def permission_catalog( - principal: ApiPrincipal = Depends(require_any_scope("admin:roles:read", "system:roles:read", "system:access:read", "system:governance:read")), -): - items = [ - PermissionItem( - scope=item.scope, - label=item.label, - description=item.description, - category=item.category, - level=item.level, - ) - for item in ALL_PERMISSIONS - if item.level == "tenant" or has_scope(principal, "system:roles:read") or has_scope(principal, "system:access:read") or has_scope(principal, "system:governance:read") - ] - return PermissionCatalogResponse(permissions=items) - - -@router.get("/tenants", response_model=TenantListResponse) -def list_tenants( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:tenants:read")), -): - tenants = session.query(Tenant).order_by(Tenant.name.asc()).all() - return TenantListResponse(tenants=[_tenant_item(session, tenant) for tenant in tenants]) - - -@router.get("/tenants/owner-candidates", response_model=TenantOwnerCandidateListResponse) -def list_tenant_owner_candidates( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:tenants:create")), -): - accounts = session.query(Account).filter(Account.is_active.is_(True)).order_by(Account.email.asc()).all() - return TenantOwnerCandidateListResponse( - accounts=[ - TenantOwnerCandidate(account_id=account.id, email=account.email, display_name=account.display_name) - for account in accounts - ] - ) - - -@router.post("/tenants", response_model=TenantAdminItem, status_code=status.HTTP_201_CREATED) -def create_tenant( - payload: TenantCreateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:tenants:create")), -): - tenant_slug = slugify(payload.slug) - if session.query(Tenant).filter(Tenant.slug == tenant_slug).count(): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="A tenant with this slug already exists") - system_defaults = get_system_settings(session) - try: - assert_tenant_governance_override_allowed(session, field="allow_custom_groups", value=payload.allow_custom_groups) - assert_tenant_governance_override_allowed(session, field="allow_custom_roles", value=payload.allow_custom_roles) - assert_tenant_governance_override_allowed(session, field="allow_api_keys", value=payload.allow_api_keys) - except AdminValidationError as exc: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc - tenant = Tenant( - slug=tenant_slug, - name=payload.name.strip(), - description=payload.description.strip() if payload.description else None, - default_locale=payload.default_locale.strip() or system_defaults.default_locale, - settings=payload.settings, - allow_custom_groups=payload.allow_custom_groups, - allow_custom_roles=payload.allow_custom_roles, - allow_api_keys=payload.allow_api_keys, - is_active=True, - ) - session.add(tenant) - session.flush() - roles = ensure_default_roles(session, tenant) - - owner_account = session.get(Account, payload.owner_account_id or principal.account.id) - if owner_account is None or not owner_account.is_active: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="The selected tenant owner account is missing or inactive.") - membership = session.query(User).filter(User.tenant_id == tenant.id, User.account_id == owner_account.id).one_or_none() - if membership is None: - membership = User( - tenant_id=tenant.id, - account_id=owner_account.id, - email=owner_account.email, - display_name=owner_account.display_name, - is_active=True, - auth_provider=owner_account.auth_provider, - password_hash=owner_account.password_hash, - ) - session.add(membership) - session.flush() - session.add(UserRoleAssignment(tenant_id=tenant.id, user_id=membership.id, role_id=roles["owner"].id)) - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="tenant.created", - object_type="tenant", - object_id=tenant.id, - scope="system", - details={"slug": tenant.slug, "name": tenant.name, "creator_account_id": principal.account.id, "owner_account_id": owner_account.id}, - ) - session.commit() - return _tenant_item(session, tenant) - - -@router.patch("/tenants/{tenant_id}", response_model=TenantAdminItem) -def update_tenant( - tenant_id: str, - payload: TenantUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - non_status_fields = {"name", "description", "default_locale", "settings", "allow_custom_groups", "allow_custom_roles", "allow_api_keys"} - if payload.model_fields_set.intersection(non_status_fields): - _require_permission(principal, "system:tenants:update") - if payload.is_active is not None: - _require_permission(principal, "system:tenants:suspend") - tenant = session.get(Tenant, tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found") - if payload.name is not None: - tenant.name = payload.name.strip() - if "description" in payload.model_fields_set: - tenant.description = payload.description.strip() if payload.description and payload.description.strip() else None - if payload.default_locale is not None: - tenant.default_locale = payload.default_locale.strip() or "en" - if payload.settings is not None: - tenant.settings = payload.settings - try: - if "allow_custom_groups" in payload.model_fields_set: - assert_tenant_governance_override_allowed(session, field="allow_custom_groups", value=payload.allow_custom_groups) - tenant.allow_custom_groups = payload.allow_custom_groups - if "allow_custom_roles" in payload.model_fields_set: - assert_tenant_governance_override_allowed(session, field="allow_custom_roles", value=payload.allow_custom_roles) - tenant.allow_custom_roles = payload.allow_custom_roles - if "allow_api_keys" in payload.model_fields_set: - assert_tenant_governance_override_allowed(session, field="allow_api_keys", value=payload.allow_api_keys) - tenant.allow_api_keys = payload.allow_api_keys - except AdminValidationError as exc: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc - if payload.is_active is not None: - if not payload.is_active and tenant.id == principal.tenant_id: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Switch to another tenant before suspending the active tenant.", - ) - tenant.is_active = payload.is_active - session.add(tenant) - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="tenant.updated", - scope="system", - object_type="tenant", - object_id=tenant.id, - details=payload.model_dump(exclude_unset=True), - ) - session.commit() - return _tenant_item(session, tenant) - - -@router.get("/tenant/settings", response_model=TenantSettingsItem) -def get_tenant_settings( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:settings:read")), -): - tenant = session.get(Tenant, principal.tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found") - return _tenant_settings_item(tenant) - - -@router.patch("/tenant/settings", response_model=TenantSettingsItem) -def update_tenant_settings( - payload: TenantSettingsUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:settings:write")), -): - tenant = session.get(Tenant, principal.tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found") - tenant.default_locale = payload.default_locale.strip() or "en" - session.add(tenant) - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="tenant.settings.updated", - object_type="tenant", - object_id=tenant.id, - details={"default_locale": tenant.default_locale}, - ) - session.commit() - return _tenant_settings_item(tenant) - - -@router.get("/users", response_model=UserListResponse) -def list_users( - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:users:read")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - users = session.query(User).filter(User.tenant_id == tenant.id).order_by(User.display_name.asc(), User.email.asc()).all() - owner_ids = tenant_owner_user_ids(session, tenant.id) - return UserListResponse(users=[_user_item(session, user, owner_ids=owner_ids) for user in users]) - - -@router.post("/users", response_model=UserCreateResponse, status_code=status.HTTP_201_CREATED) -def create_user( - payload: UserCreateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - _require_permission(principal, "admin:users:create") - if payload.group_ids: - _require_permission(principal, "admin:groups:manage_members") - if payload.role_ids: - _require_permission(principal, "admin:roles:assign") - tenant = _resolve_tenant(session, principal, tenant_id) - try: - result = create_membership( - session, - tenant=tenant, - email=payload.email, - display_name=payload.display_name, - password=payload.password, - password_reset_required=payload.password_reset_required, - is_active=payload.is_active, - ) - set_user_groups(session, user=result.user, group_ids=payload.group_ids) - assert_can_delegate_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids, tenant_id=tenant.id) - set_user_roles(session, user=result.user, role_ids=payload.role_ids) - assert_tenant_owner_exists(session, tenant.id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="user.created", - object_type="user", - object_id=result.user.id, - details={ - "email": result.account.email, - "account_created": result.account_created, - "group_ids": payload.group_ids, - "role_ids": payload.role_ids, - }, - ) - session.commit() - return UserCreateResponse( - user=_user_item(session, result.user), - account_created=result.account_created, - temporary_password=result.temporary_password, - ) - - -@router.patch("/users/{user_id}", response_model=UserAdminItem) -def update_user( - user_id: str, - payload: UserUpdateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - if "display_name" in payload.model_fields_set: - _require_permission(principal, "admin:users:update") - if payload.is_active is not None: - _require_permission(principal, "admin:users:suspend") - if payload.group_ids is not None: - _require_permission(principal, "admin:groups:manage_members") - if payload.role_ids is not None: - _require_permission(principal, "admin:roles:assign") - tenant = _resolve_tenant(session, principal, tenant_id) - user = session.query(User).filter(User.id == user_id, User.tenant_id == tenant.id).one_or_none() - if user is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found") - try: - if payload.display_name is not None: - user.display_name = payload.display_name.strip() or None - if payload.is_active is not None: - user.is_active = payload.is_active - session.add(user) - if payload.group_ids is not None: - set_user_groups(session, user=user, group_ids=payload.group_ids) - if payload.role_ids is not None: - assert_can_delegate_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids, tenant_id=tenant.id) - set_user_roles(session, user=user, role_ids=payload.role_ids) - assert_tenant_owner_exists(session, tenant.id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="user.updated", - object_type="user", - object_id=user.id, - details=payload.model_dump(exclude_none=True), - ) - session.commit() - return _user_item(session, user) - - -@router.get("/groups", response_model=GroupListResponse) -def list_groups( - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:groups:read")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - groups = session.query(Group).filter(Group.tenant_id == tenant.id).order_by(Group.name.asc()).all() - return GroupListResponse(groups=[_group_summary(session, group) for group in groups]) - - -@router.post("/groups", response_model=GroupSummary, status_code=status.HTTP_201_CREATED) -def create_group( - payload: GroupCreateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - _require_permission(principal, "admin:groups:write") - if payload.member_ids: - _require_permission(principal, "admin:groups:manage_members") - if payload.role_ids: - _require_permission(principal, "admin:roles:assign") - tenant = _resolve_tenant(session, principal, tenant_id) - try: - assert_custom_groups_allowed(session, tenant) - except AdminConflictError as exc: - raise _http_admin_error(exc) from exc - group_slug = slugify(payload.slug) - if session.query(Group).filter(Group.tenant_id == tenant.id, Group.slug == group_slug).count(): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="A group with this slug already exists") - group = Group( - tenant_id=tenant.id, - slug=group_slug, - name=payload.name.strip(), - description=payload.description.strip() if payload.description else None, - is_active=payload.is_active, - ) - session.add(group) - session.flush() - try: - set_group_members(session, group=group, user_ids=payload.member_ids) - assert_can_delegate_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids, tenant_id=tenant.id) - set_group_roles(session, group=group, role_ids=payload.role_ids) - assert_tenant_owner_exists(session, tenant.id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="group.created", - object_type="group", - object_id=group.id, - details={"slug": group.slug, "member_ids": payload.member_ids, "role_ids": payload.role_ids}, - ) - session.commit() - return _group_summary(session, group) - - -@router.patch("/groups/{group_id}", response_model=GroupSummary) -def update_group( - group_id: str, - payload: GroupUpdateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - if any(field in payload.model_fields_set for field in ("name", "description", "is_active")): - _require_permission(principal, "admin:groups:write") - if payload.member_ids is not None: - _require_permission(principal, "admin:groups:manage_members") - if payload.role_ids is not None: - _require_permission(principal, "admin:roles:assign") - tenant = _resolve_tenant(session, principal, tenant_id) - group = session.query(Group).filter(Group.id == group_id, Group.tenant_id == tenant.id).one_or_none() - if group is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Group not found") - try: - ensure_group_mutation_allowed(group, requested_active=payload.is_active) - if group.system_template_id is None and payload.name is not None: - group.name = payload.name.strip() - if group.system_template_id is None and "description" in payload.model_fields_set: - group.description = payload.description.strip() if payload.description and payload.description.strip() else None - if payload.is_active is not None: - group.is_active = payload.is_active - session.add(group) - if payload.member_ids is not None: - set_group_members(session, group=group, user_ids=payload.member_ids) - if payload.role_ids is not None: - assert_can_delegate_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids, tenant_id=tenant.id) - set_group_roles(session, group=group, role_ids=payload.role_ids) - assert_tenant_owner_exists(session, tenant.id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="group.updated", - object_type="group", - object_id=group.id, - details=payload.model_dump(exclude_unset=True), - ) - session.commit() - return _group_summary(session, group) - - -@router.get("/roles", response_model=RoleListResponse) -def list_roles( - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:roles:read")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - ensure_default_roles(session, tenant) - session.commit() - roles = session.query(Role).filter(Role.tenant_id == tenant.id).order_by(Role.is_builtin.desc(), Role.name.asc()).all() - return RoleListResponse(roles=[_role_summary(session, role) for role in roles]) - - -@router.post("/roles", response_model=RoleSummary, status_code=status.HTTP_201_CREATED) -def create_role( - payload: RoleCreateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:roles:write")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - try: - assert_custom_roles_allowed(session, tenant) - assert_can_delegate_tenant_permissions(principal.scopes, payload.permissions) - role = create_custom_role( - session, - tenant=tenant, - slug=payload.slug, - name=payload.name, - description=payload.description, - permissions=payload.permissions, - ) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="role.created", - object_type="role", - object_id=role.id, - details={"slug": role.slug, "permissions": role.permissions}, - ) - session.commit() - return _role_summary(session, role) - - -@router.patch("/roles/{role_id}", response_model=RoleSummary) -def update_role( - role_id: str, - payload: RoleUpdateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:roles:write")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - role = session.query(Role).filter(Role.id == role_id, Role.tenant_id == tenant.id).one_or_none() - if role is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Role not found") - try: - ensure_role_mutation_allowed(role, requested_assignable=payload.is_assignable) - if payload.permissions is not None: - assert_can_delegate_tenant_permissions(principal.scopes, payload.permissions) - update_custom_role( - session, - role=role, - name=payload.name, - description=payload.description, - permissions=payload.permissions, - is_assignable=payload.is_assignable, - ) - assert_tenant_owner_exists(session, tenant.id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="role.updated", - object_type="role", - object_id=role.id, - details=payload.model_dump(), - ) - session.commit() - return _role_summary(session, role) - - -@router.delete("/roles/{role_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_role( - role_id: str, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:roles:write")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - role = session.query(Role).filter(Role.id == role_id, Role.tenant_id == tenant.id).one_or_none() - if role is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Role not found") - try: - if role.system_template_id: - raise AdminConflictError("Centrally managed roles can only be removed from System administration.") - role_slug = role.slug - delete_custom_role(session, role) - assert_tenant_owner_exists(session, tenant.id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="role.deleted", - object_type="role", - object_id=role_id, - details={"slug": role_slug}, - ) - session.commit() - return None - - -@router.get("/system/roles", response_model=RoleListResponse) -def list_system_roles( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_any_scope("system:roles:read", "system:access:read")), -): - ensure_default_roles(session, None) - session.commit() - roles = session.query(Role).filter(Role.tenant_id.is_(None)).order_by(Role.name.asc()).all() - return RoleListResponse(roles=[_role_summary(session, role) for role in roles]) - - -@router.post("/system/roles", response_model=RoleSummary, status_code=status.HTTP_201_CREATED) -def create_system_role_endpoint( - payload: RoleCreateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:roles:write")), -): - try: - assert_can_delegate_system_permissions(principal.scopes, payload.permissions) - role = create_system_role( - session, - slug=payload.slug, - name=payload.name, - description=payload.description, - permissions=payload.permissions, - ) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="system_role.created", scope="system", object_type="role", object_id=role.id, - details={"slug": role.slug, "permissions": role.permissions}, - ) - session.commit() - return _role_summary(session, role) - - -@router.patch("/system/roles/{role_id}", response_model=RoleSummary) -def update_system_role_endpoint( - role_id: str, - payload: RoleUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:roles:write")), -): - role = session.query(Role).filter(Role.id == role_id, Role.tenant_id.is_(None)).one_or_none() - if role is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="System role not found") - try: - assert_can_delegate_system_permissions(principal.scopes, payload.permissions) - update_system_role( - session, - role=role, - name=payload.name, - description=payload.description, - permissions=payload.permissions, - is_assignable=payload.is_assignable, - ) - assert_system_owner_exists(session) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="system_role.updated", scope="system", object_type="role", object_id=role.id, - details=payload.model_dump(), - ) - session.commit() - return _role_summary(session, role) - - -@router.delete("/system/roles/{role_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_system_role_endpoint( - role_id: str, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:roles:write")), -): - role = session.query(Role).filter(Role.id == role_id, Role.tenant_id.is_(None)).one_or_none() - if role is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="System role not found") - try: - role_slug = role.slug - delete_system_role(session, role) - assert_system_owner_exists(session) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="system_role.deleted", scope="system", object_type="role", object_id=role_id, - details={"slug": role_slug}, - ) - session.commit() - return None - - -@router.get("/system/accounts", response_model=SystemAccountListResponse) -def list_system_accounts( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_any_scope("system:accounts:read", "system:access:read")), -): - ensure_default_roles(session, None) - session.commit() - system_roles = session.query(Role).filter(Role.tenant_id.is_(None)).order_by(Role.name.asc()).all() - accounts = session.query(Account).order_by(Account.email.asc()).all() - return SystemAccountListResponse( - accounts=[_system_account_item(session, account) for account in accounts], - roles=[_role_summary(session, role) for role in system_roles], - ) - - -@router.patch("/system/accounts/{account_id}", response_model=SystemAccountItem) -def update_system_account( - account_id: str, - payload: SystemAccountUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - if "display_name" in payload.model_fields_set and not has_scope(principal, "system:accounts:update"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:accounts:update") - if payload.is_active is not None and not has_scope(principal, "system:accounts:suspend"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:accounts:suspend") - if payload.role_ids is not None: - _require_system_role_assignment(principal) - account = session.get(Account, account_id) - if account is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Account not found") - if "display_name" in payload.model_fields_set: - account.display_name = payload.display_name.strip() if payload.display_name else None - if payload.is_active is not None: - account.is_active = payload.is_active - session.add(account) - try: - if payload.role_ids is not None: - _require_system_role_assignment(principal) - assert_can_delegate_system_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids) - set_system_roles(session, account=account, role_ids=payload.role_ids) - else: - session.flush() - assert_system_owner_exists(session) - if not account.is_active: - tenant_ids = [ - row[0] - for row in session.query(User.tenant_id) - .join(Tenant, Tenant.id == User.tenant_id) - .filter(User.account_id == account.id, User.is_active.is_(True), Tenant.is_active.is_(True)) - .distinct() - .all() - ] - for tenant_id in tenant_ids: - assert_tenant_owner_exists(session, tenant_id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, - principal, - action="system_account.updated", - scope="system", - object_type="account", - object_id=account.id, - details=payload.model_dump(exclude_unset=True), - ) - session.commit() - return _system_account_item(session, account) - - -@router.put("/system/accounts/{account_id}/roles", response_model=SystemAccountItem) -def update_system_account_roles( - account_id: str, - payload: SystemAccountRolesUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_any_scope("system:roles:assign", "system:access:assign")), -): - account = session.get(Account, account_id) - if account is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Account not found") - try: - _require_system_role_assignment(principal) - assert_can_delegate_system_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids) - set_system_roles(session, account=account, role_ids=payload.role_ids) - assert_system_owner_exists(session) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, - principal, - action="system_access.updated", - scope="system", - object_type="account", - object_id=account.id, - details={"role_ids": payload.role_ids}, - ) - session.commit() - return _system_account_item(session, account) - - -@router.post("/system/accounts", response_model=SystemAccountCreateResponse, status_code=status.HTTP_201_CREATED) -def create_system_account( - payload: SystemAccountCreateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:accounts:create")), -): - try: - account, created, temporary_password = get_or_create_account( - session, - email=payload.email, - display_name=payload.display_name, - password=payload.password, - password_reset_required=payload.password_reset_required, - ) - if not created: - raise AdminConflictError("A global account with this email already exists.") - account.is_active = payload.is_active - if payload.role_ids: - _require_system_role_assignment(principal) - assert_can_delegate_system_roles(session, actor_scopes=principal.scopes, role_ids=payload.role_ids) - set_system_roles(session, account=account, role_ids=payload.role_ids) - if payload.memberships: - _require_permission(principal, "system:access:assign") - _set_system_memberships(session, account, [item.model_dump() for item in payload.memberships]) - assert_system_owner_exists(session) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="system_account.created", scope="system", object_type="account", object_id=account.id, - details={"email": account.email, "tenant_ids": [item.tenant_id for item in payload.memberships], "role_ids": payload.role_ids}, - ) - session.commit() - return SystemAccountCreateResponse(account=_system_account_item(session, account), temporary_password=temporary_password) - - -@router.put("/system/accounts/{account_id}/memberships", response_model=SystemAccountItem) -def update_system_account_memberships( - account_id: str, - payload: SystemAccountMembershipsUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:accounts:update")), -): - account = session.get(Account, account_id) - if account is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Account not found") - try: - _require_permission(principal, "system:access:assign") - _set_system_memberships(session, account, [item.model_dump() for item in payload.memberships]) - for tenant_id in {item.tenant_id for item in payload.memberships}: - assert_tenant_owner_exists(session, tenant_id) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="system_memberships.updated", scope="system", object_type="account", object_id=account.id, - details={"tenant_ids": [item.tenant_id for item in payload.memberships]}, - ) - session.commit() - return _system_account_item(session, account) - - -def _set_system_memberships(session: Session, account: Account, requested: list[dict]) -> None: - desired = {item["tenant_id"]: item for item in requested} - existing = {item.tenant_id: item for item in session.query(User).filter(User.account_id == account.id).all()} - affected = set(existing) | set(desired) - for tenant_id, user in existing.items(): - if tenant_id not in desired: - user.is_active = False - session.add(user) - for tenant_id, item in desired.items(): - tenant = session.get(Tenant, tenant_id) - if tenant is None: - raise AdminValidationError(f"Unknown tenant: {tenant_id}") - user = existing.get(tenant_id) - if user is None: - user = User( - tenant_id=tenant.id, - account_id=account.id, - email=account.email, - display_name=account.display_name, - is_active=bool(item.get("is_active", True)), - auth_provider=account.auth_provider, - password_hash=account.password_hash, - ) - session.add(user) - session.flush() - else: - user.is_active = bool(item.get("is_active", True)) - user.email = account.email - session.add(user) - set_user_roles(session, user=user, role_ids=item.get("role_ids", [])) - set_user_groups(session, user=user, group_ids=item.get("group_ids", [])) - session.flush() - for tenant_id in affected: - tenant = session.get(Tenant, tenant_id) - if tenant and tenant.is_active: - assert_tenant_owner_exists(session, tenant_id) - - -@router.get("/system/settings", response_model=SystemSettingsItem) -def read_system_settings( - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:settings:read")), -): - item = get_system_settings(session) - session.commit() - policy = privacy_policy_from_settings(item) - return SystemSettingsItem( - default_locale=item.default_locale, - allow_tenant_custom_groups=item.allow_tenant_custom_groups, - allow_tenant_custom_roles=item.allow_tenant_custom_roles, - allow_tenant_api_keys=item.allow_tenant_api_keys, - privacy_retention_policy=PrivacyRetentionPolicyItem.model_validate(policy.model_dump(mode="json")), - settings=item.settings or {}, - ) - - -@router.patch("/system/settings", response_model=SystemSettingsItem) -def write_system_settings( - payload: SystemSettingsUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:settings:write")), -): - item = get_system_settings(session) - item.default_locale = payload.default_locale.strip() or "en" - item.allow_tenant_custom_groups = payload.allow_tenant_custom_groups - item.allow_tenant_custom_roles = payload.allow_tenant_custom_roles - item.allow_tenant_api_keys = payload.allow_tenant_api_keys - if payload.privacy_retention_policy is not None: - set_privacy_policy(item, payload.privacy_retention_policy.model_dump(mode="json")) - session.add(item) - audit_from_principal( - session, principal, action="system_settings.updated", scope="system", object_type="system_settings", object_id=item.id, - details=payload.model_dump(), - ) - session.commit() - return read_system_settings(session=session, principal=principal) - - -def _require_privacy_policy_read(principal: ApiPrincipal, scope_type: str) -> None: - if scope_type == "system": - _require_permission(principal, "system:settings:read") - else: - _require_permission(principal, "admin:policies:read") - - -def _require_privacy_policy_write(principal: ApiPrincipal, scope_type: str) -> None: - if scope_type == "system": - _require_permission(principal, "system:settings:write") - else: - _require_permission(principal, "admin:policies:write") - - -@router.get("/privacy-retention/policies/{scope_type}", response_model=PrivacyRetentionPolicyScopeResponse) -def read_privacy_retention_policy( - scope_type: str, - scope_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - clean_scope = scope_type.strip().casefold() - _require_privacy_policy_read(principal, clean_scope) - try: - policy = get_privacy_policy_for_scope(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id) - effective = _effective_privacy_policy_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id) - parent = _parent_privacy_policy_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id) - return PrivacyRetentionPolicyScopeResponse( - scope_type=clean_scope, - scope_id=scope_id, - policy=policy, - effective_policy=PrivacyRetentionPolicyItem.model_validate(effective.model_dump(mode="json")), - parent_policy=PrivacyRetentionPolicyItem.model_validate(parent.model_dump(mode="json")) if parent else None, - effective_policy_sources=_effective_privacy_policy_sources_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id), - parent_policy_sources=_parent_privacy_policy_sources_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id), - ) - except PrivacyPolicyError as exc: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc - - -@router.put("/privacy-retention/policies/{scope_type}", response_model=PrivacyRetentionPolicyScopeResponse) -def write_privacy_retention_policy( - scope_type: str, - payload: PrivacyRetentionPolicyScopeRequest, - scope_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(get_api_principal), -): - clean_scope = scope_type.strip().casefold() - _require_privacy_policy_write(principal, clean_scope) - try: - policy = set_privacy_policy_for_scope( - session, - tenant_id=principal.tenant_id, - scope_type=clean_scope, - scope_id=scope_id, - policy=payload.policy.model_dump(mode="json", exclude_none=True), - ) - session.commit() - effective = _effective_privacy_policy_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id) - parent = _parent_privacy_policy_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id) - return PrivacyRetentionPolicyScopeResponse( - scope_type=clean_scope, - scope_id=scope_id, - policy=policy, - effective_policy=PrivacyRetentionPolicyItem.model_validate(effective.model_dump(mode="json")), - parent_policy=PrivacyRetentionPolicyItem.model_validate(parent.model_dump(mode="json")) if parent else None, - effective_policy_sources=_effective_privacy_policy_sources_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id), - parent_policy_sources=_parent_privacy_policy_sources_for_response(session, tenant_id=principal.tenant_id, scope_type=clean_scope, scope_id=scope_id), - ) - except PrivacyPolicyError as exc: - session.rollback() - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc - - - -def _parent_privacy_policy_sources_for_response(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None): - if scope_type == "system": - return [] - return parent_privacy_policy_sources(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id or (tenant_id if scope_type == "tenant" else None)) - - -def _effective_privacy_policy_sources_for_response(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None): - if scope_type == "system": - return effective_privacy_policy_sources(session) - if scope_type == "tenant": - return effective_privacy_policy_sources(session, tenant_id=scope_id or tenant_id) - if scope_type == "campaign" and scope_id: - return effective_privacy_policy_sources(session, campaign_id=scope_id) - if scope_type == "user" and scope_id: - return effective_privacy_policy_sources(session, tenant_id=tenant_id, owner_user_id=scope_id) - if scope_type == "group" and scope_id: - return effective_privacy_policy_sources(session, tenant_id=tenant_id, owner_group_id=scope_id) - return effective_privacy_policy_sources(session, tenant_id=tenant_id) - -def _parent_privacy_policy_for_response(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None): - if scope_type == "system": - return None - return parent_privacy_policy(session, tenant_id=tenant_id, scope_type=scope_type, scope_id=scope_id or (tenant_id if scope_type == "tenant" else None)) - - -def _effective_privacy_policy_for_response(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None): - if scope_type == "system": - return effective_privacy_policy(session) - if scope_type == "tenant": - return effective_privacy_policy(session, tenant_id=scope_id or tenant_id) - if scope_type == "campaign" and scope_id: - return effective_privacy_policy(session, campaign_id=scope_id) - if scope_type == "user" and scope_id: - return effective_privacy_policy(session, tenant_id=tenant_id, owner_user_id=scope_id) - if scope_type == "group" and scope_id: - return effective_privacy_policy(session, tenant_id=tenant_id, owner_group_id=scope_id) - return effective_privacy_policy(session, tenant_id=tenant_id) - - -@router.post("/system/retention/run", response_model=RetentionRunResponse) -def run_retention_policy( - payload: RetentionRunRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:settings:write")), -): - result = apply_retention_policy(session, dry_run=payload.dry_run) - audit_from_principal( - session, - principal, - action="retention_policy.run", - scope="system", - object_type="retention_policy", - object_id="global", - details={"dry_run": payload.dry_run, "counts": result.get("counts")}, - ) - session.commit() - return RetentionRunResponse(result=result) - - -@router.get("/system/governance-templates", response_model=GovernanceTemplateListResponse) -def list_governance_templates( - kind: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:governance:read")), -): - query = session.query(GovernanceTemplate) - if kind: - query = query.filter(GovernanceTemplate.kind == kind) - items = query.order_by(GovernanceTemplate.kind.asc(), GovernanceTemplate.name.asc()).all() - return GovernanceTemplateListResponse(templates=[_governance_template_item(session, item) for item in items]) - - -@router.post("/system/governance-templates", response_model=GovernanceTemplateItem, status_code=status.HTTP_201_CREATED) -def create_governance_template( - payload: GovernanceTemplateCreateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:governance:write")), -): - try: - item = create_template( - session, - kind=payload.kind, - slug=payload.slug, - name=payload.name, - description=payload.description, - permissions=payload.permissions, - is_active=payload.is_active, - assignments=[assignment.model_dump() for assignment in payload.assignments], - ) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="governance_template.created", scope="system", object_type=f"{payload.kind}_template", object_id=item.id, - details={"slug": item.slug, "tenant_ids": [assignment.tenant_id for assignment in payload.assignments]}, - ) - session.commit() - return _governance_template_item(session, item) - - -@router.patch("/system/governance-templates/{template_id}", response_model=GovernanceTemplateItem) -def update_governance_template( - template_id: str, - payload: GovernanceTemplateUpdateRequest, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:governance:write")), -): - item = session.get(GovernanceTemplate, template_id) - if item is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Governance template not found") - try: - update_template( - session, item, - name=payload.name, - description=payload.description, - permissions=payload.permissions, - is_active=payload.is_active, - assignments=[assignment.model_dump() for assignment in payload.assignments], - ) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="governance_template.updated", scope="system", object_type=f"{item.kind}_template", object_id=item.id, - details={"tenant_ids": [assignment.tenant_id for assignment in payload.assignments]}, - ) - session.commit() - return _governance_template_item(session, item) - - -@router.delete("/system/governance-templates/{template_id}", status_code=status.HTTP_204_NO_CONTENT) -def remove_governance_template( - template_id: str, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("system:governance:write")), -): - item = session.get(GovernanceTemplate, template_id) - if item is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Governance template not found") - kind = item.kind - try: - delete_template(session, item) - except (AdminConflictError, AdminValidationError) as exc: - raise _http_admin_error(exc) from exc - audit_from_principal( - session, principal, action="governance_template.deleted", scope="system", object_type=f"{kind}_template", object_id=template_id, - details={}, - ) - session.commit() - return None - - -@router.get("/api-keys", response_model=ApiKeyListResponse) -def list_api_keys( - tenant_id: str | None = Query(default=None), - include_revoked: bool = Query(default=False), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:api_keys:read")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - query = session.query(ApiKey).filter(ApiKey.tenant_id == tenant.id) - if not include_revoked: - query = query.filter(ApiKey.revoked_at.is_(None)) - keys = query.order_by(ApiKey.created_at.desc()).all() - return ApiKeyListResponse(api_keys=[_api_key_item(session, item) for item in keys]) - - -@router.post("/api-keys", response_model=AdminApiKeyCreateResponse, status_code=status.HTTP_201_CREATED) -def create_tenant_api_key( - payload: AdminApiKeyCreateRequest, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:api_keys:create")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - try: - assert_api_keys_allowed(session, tenant) - except AdminConflictError as exc: - raise _http_admin_error(exc) from exc - user_id = payload.user_id or principal.user.id - user = session.query(User).filter(User.id == user_id, User.tenant_id == tenant.id, User.is_active.is_(True)).one_or_none() - if user is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Active user not found") - user_scopes = collect_user_scopes(session, user, include_system=False) - requested = payload.scopes or ["campaign:read"] - invalid = [scope for scope in requested if scope.startswith("system:") or not scopes_grant(user_scopes, scope)] - if invalid: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"API key scopes exceed the user's tenant permissions: {', '.join(invalid)}", - ) - created = create_api_key( - session, - user=user, - name=payload.name.strip(), - scopes=sorted(set(requested)), - expires_at=payload.expires_at, - ) - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="api_key.created", - object_type="api_key", - object_id=created.model.id, - details={"name": created.model.name, "prefix": created.model.prefix, "scopes": created.model.scopes, "owner_user_id": user.id}, - ) - session.commit() - return AdminApiKeyCreateResponse(**_api_key_item(session, created.model).model_dump(), secret=created.secret) - - -@router.post("/api-keys/{api_key_id}/revoke", response_model=ApiKeyAdminItem) -def revoke_api_key( - api_key_id: str, - tenant_id: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("admin:api_keys:revoke")), -): - tenant = _resolve_tenant(session, principal, tenant_id) - item = session.query(ApiKey).filter(ApiKey.id == api_key_id, ApiKey.tenant_id == tenant.id).one_or_none() - if item is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="API key not found") - if item.revoked_at is None: - item.revoked_at = utc_now() - session.add(item) - audit_event( - session, - tenant_id=tenant.id, - user_id=principal.user.id, - action="api_key.revoked", - object_type="api_key", - object_id=item.id, - details={"name": item.name, "prefix": item.prefix}, - ) - session.commit() - return _api_key_item(session, item) - - -def _parse_audit_filter(value: str | None) -> tuple[str, str]: - if not value: - return "contains", "" - if ":" not in value: - return "contains", value.strip() - operator, raw = value.split(":", 1) - if operator not in {"contains", "eq", "before", "after", "gt", "gte", "lt", "lte"}: - return "contains", value.strip() - return operator, raw.strip() - - -def _audit_text_filter(column, raw: str | None): - operator, value = _parse_audit_filter(raw) - if not value: - return None - normalized = value.casefold() - text = func.lower(func.coalesce(column, "")) - return text == normalized if operator == "eq" else text.contains(normalized) - - -def _audit_time_filter(raw: str | None): - operator, value = _parse_audit_filter(raw) - if not value: - return None - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Invalid audit date filter.") from exc - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - else: - parsed = parsed.astimezone(timezone.utc) - if operator == "eq": - if "T" not in value and " " not in value: - return (AuditLog.created_at >= parsed) & (AuditLog.created_at < parsed + timedelta(days=1)) - return AuditLog.created_at == parsed - if operator in {"before", "lt"}: - return AuditLog.created_at < parsed - if operator == "lte": - return AuditLog.created_at <= parsed - if operator in {"after", "gt"}: - return AuditLog.created_at > parsed - if operator == "gte": - return AuditLog.created_at >= parsed - return AuditLog.created_at == parsed - - -@router.get("/audit", response_model=AuditAdminListResponse) -def list_admin_audit( - tenant_id: str | None = Query(default=None), - all_tenants: bool = Query(default=False), - audit_scope: str | None = Query(default=None, alias="scope"), - limit: int = Query(default=100, ge=1, le=500), - offset: int = Query(default=0, ge=0), - page: int | None = Query(default=None, ge=1), - page_size: int | None = Query(default=None, ge=1, le=500), - sort_by: str = Query(default="time"), - sort_direction: str = Query(default="desc"), - filter_time: str | None = Query(default=None), - filter_actor: str | None = Query(default=None), - filter_action: str | None = Query(default=None), - filter_object: str | None = Query(default=None), - filter_tenant: str | None = Query(default=None), - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_any_scope("audit:read", "system:audit:read")), -): - effective_scope = audit_scope or ("all" if all_tenants else "tenant") - if effective_scope not in {"tenant", "system", "all"}: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Audit scope must be tenant, system or all.") - if sort_by not in {"time", "actor", "action", "object", "tenant"}: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Unsupported audit sort column.") - if sort_direction not in {"asc", "desc"}: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Audit sort direction must be asc or desc.") - - query = ( - session.query(AuditLog) - .outerjoin(User, AuditLog.user_id == User.id) - .outerjoin(Account, User.account_id == Account.id) - ) - if effective_scope != "all": - query = query.filter(AuditLog.scope == effective_scope) - if effective_scope == "system": - if not has_scope(principal, "system:audit:read"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:audit:read") - elif effective_scope == "all" or all_tenants: - if not has_scope(principal, "system:audit:read"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: system:audit:read") - else: - if not has_scope(principal, "audit:read"): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: audit:read") - tenant = _resolve_tenant(session, principal, tenant_id) - query = query.filter(AuditLog.tenant_id == tenant.id) - - object_text = func.coalesce(AuditLog.object_type, "") + " " + func.coalesce(AuditLog.object_id, "") - filters = [ - _audit_time_filter(filter_time), - _audit_text_filter(func.coalesce(Account.email, "System"), filter_actor), - _audit_text_filter(AuditLog.action, filter_action), - _audit_text_filter(object_text, filter_object), - _audit_text_filter(AuditLog.tenant_id, filter_tenant), - ] - for condition in filters: - if condition is not None: - query = query.filter(condition) - - total = query.count() - effective_page_size = page_size or limit - pages = max(1, (total + effective_page_size - 1) // effective_page_size) - if page is not None or page_size is not None: - effective_page = min(page or 1, pages) - effective_offset = (effective_page - 1) * effective_page_size - else: - effective_page = offset // effective_page_size + 1 - effective_offset = offset - - sort_columns = { - "time": AuditLog.created_at, - "actor": func.coalesce(Account.email, "System"), - "action": AuditLog.action, - "object": object_text, - "tenant": func.coalesce(AuditLog.tenant_id, ""), - } - sort_column = sort_columns[sort_by] - order = sort_column.asc() if sort_direction == "asc" else sort_column.desc() - rows = query.order_by(order, AuditLog.id.desc()).offset(effective_offset).limit(effective_page_size).all() - account_by_user_id = { - user.id: session.get(Account, user.account_id) - for user in session.query(User).filter(User.id.in_({row.user_id for row in rows if row.user_id})).all() - } if rows else {} - - return AuditAdminListResponse( - total=total, - page=effective_page, - page_size=effective_page_size, - pages=pages, - items=[ - AuditAdminItem( - id=row.id, - scope=row.scope, - tenant_id=row.tenant_id, - actor_email=account_by_user_id[row.user_id].email if row.user_id in account_by_user_id and account_by_user_id[row.user_id] else None, - action=row.action, - object_type=row.object_type, - object_id=row.object_id, - details=row.details or {}, - created_at=row.created_at, - ) - for row in rows - ], - ) diff --git a/src/govoplan_core/api/v1/admin_schemas.py b/src/govoplan_core/api/v1/admin_schemas.py deleted file mode 100644 index c122838..0000000 --- a/src/govoplan_core/api/v1/admin_schemas.py +++ /dev/null @@ -1,510 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator - -RETENTION_DAY_KEYS = ( - "raw_campaign_json_retention_days", - "generated_eml_retention_days", - "stored_report_detail_retention_days", - "mock_mailbox_retention_days", - "audit_detail_retention_days", -) -RETENTION_POLICY_FIELD_KEYS = ( - "store_raw_campaign_json", - *RETENTION_DAY_KEYS, - "audit_detail_level", -) - - -def default_allow_lower_level_limits() -> dict[str, bool]: - return {key: True for key in RETENTION_POLICY_FIELD_KEYS} - - -def normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> dict[str, bool] | None: - if value in (None, ""): - return default_allow_lower_level_limits() if fill_defaults else None - if not isinstance(value, dict): - raise ValueError("allow_lower_level_limits must be an object") - normalized = default_allow_lower_level_limits() if fill_defaults else {} - for key, allowed in value.items(): - clean_key = str(key) - if clean_key not in RETENTION_POLICY_FIELD_KEYS: - raise ValueError(f"Unknown retention policy field: {clean_key}") - normalized[clean_key] = bool(allowed) - return normalized - - - -class PermissionItem(BaseModel): - scope: str - label: str - description: str - category: str - level: Literal["tenant", "system"] - - -class PermissionCatalogResponse(BaseModel): - permissions: list[PermissionItem] - - -class AdminOverviewResponse(BaseModel): - active_tenant_id: str - active_tenant_name: str - tenant_count: int | None = None - system_account_count: int | None = None - system_group_template_count: int | None = None - system_role_template_count: int | None = None - user_count: int - active_user_count: int - group_count: int - role_count: int - active_api_key_count: int - capabilities: list[str] = Field(default_factory=list) - - -class TenantAdminItem(BaseModel): - id: str - slug: str = Field(min_length=1, max_length=100) - name: str = Field(min_length=1, max_length=255) - description: str | None = None - default_locale: str = Field(default="en", min_length=1, max_length=20) - settings: dict[str, Any] = Field(default_factory=dict) - allow_custom_groups: bool | None = None - allow_custom_roles: bool | None = None - allow_api_keys: bool | None = None - effective_governance: dict[str, bool] = Field(default_factory=dict) - is_active: bool - counts: dict[str, int] = Field(default_factory=dict) - created_at: datetime - updated_at: datetime - - -class TenantListResponse(BaseModel): - tenants: list[TenantAdminItem] - - -class TenantOwnerCandidate(BaseModel): - account_id: str - email: str - display_name: str | None = None - - -class TenantOwnerCandidateListResponse(BaseModel): - accounts: list[TenantOwnerCandidate] - - -class TenantCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - slug: str - name: str - owner_account_id: str | None = None - description: str | None = None - default_locale: str = "en" - settings: dict[str, Any] = Field(default_factory=dict) - allow_custom_groups: bool | None = None - allow_custom_roles: bool | None = None - allow_api_keys: bool | None = None - - -class TenantUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str | None = Field(default=None, min_length=1, max_length=255) - description: str | None = None - default_locale: str | None = Field(default=None, min_length=1, max_length=20) - settings: dict[str, Any] | None = None - allow_custom_groups: bool | None = None - allow_custom_roles: bool | None = None - allow_api_keys: bool | None = None - is_active: bool | None = None - - -class TenantSettingsItem(BaseModel): - id: str - slug: str - name: str - default_locale: str = Field(default="en", min_length=1, max_length=20) - settings: dict[str, Any] = Field(default_factory=dict) - - -class TenantSettingsUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - default_locale: str = Field(min_length=1, max_length=20) - - -class RoleSummary(BaseModel): - id: str - slug: str = Field(min_length=1, max_length=100) - name: str = Field(min_length=1, max_length=255) - description: str | None = None - permissions: list[str] = Field(default_factory=list) - effective_permission_count: int = 0 - is_builtin: bool = False - is_assignable: bool = True - user_assignments: int = 0 - group_assignments: int = 0 - level: Literal["tenant", "system"] = "tenant" - system_template_id: str | None = None - system_required: bool = False - - -class GroupSummary(BaseModel): - id: str - slug: str = Field(min_length=1, max_length=100) - name: str = Field(min_length=1, max_length=255) - description: str | None = None - is_active: bool = True - member_count: int = 0 - member_ids: list[str] = Field(default_factory=list) - roles: list[RoleSummary] = Field(default_factory=list) - created_at: datetime - updated_at: datetime - system_template_id: str | None = None - system_required: bool = False - - -class UserAdminItem(BaseModel): - id: str - account_id: str - tenant_id: str - email: str = Field(min_length=3, max_length=320) - display_name: str | None = Field(default=None, max_length=255) - is_active: bool - account_is_active: bool - password_reset_required: bool = False - last_login_at: datetime | None = None - groups: list[GroupSummary] = Field(default_factory=list) - roles: list[RoleSummary] = Field(default_factory=list) - effective_scopes: list[str] = Field(default_factory=list) - is_owner: bool = False - is_last_active_owner: bool = False - created_at: datetime - updated_at: datetime - - -class UserListResponse(BaseModel): - users: list[UserAdminItem] - - -class UserCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - email: str - display_name: str | None = None - password: str | None = Field(default=None, min_length=10) - password_reset_required: bool = True - is_active: bool = True - group_ids: list[str] = Field(default_factory=list) - role_ids: list[str] = Field(default_factory=list) - - -class UserCreateResponse(BaseModel): - user: UserAdminItem - account_created: bool - temporary_password: str | None = None - - -class UserUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - display_name: str | None = Field(default=None, max_length=255) - is_active: bool | None = None - group_ids: list[str] | None = None - role_ids: list[str] | None = None - - -class GroupListResponse(BaseModel): - groups: list[GroupSummary] - - -class GroupCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - slug: str - name: str - description: str | None = None - is_active: bool = True - member_ids: list[str] = Field(default_factory=list) - role_ids: list[str] = Field(default_factory=list) - - -class GroupUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str | None = Field(default=None, min_length=1, max_length=255) - description: str | None = None - is_active: bool | None = None - member_ids: list[str] | None = None - role_ids: list[str] | None = None - - -class RoleListResponse(BaseModel): - roles: list[RoleSummary] - - -class RoleCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - slug: str - name: str = Field(min_length=1, max_length=255) - description: str | None = None - permissions: list[str] = Field(default_factory=list) - - -class RoleUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str - description: str | None = None - permissions: list[str] = Field(default_factory=list) - is_assignable: bool = True - - -class SystemAccountItem(BaseModel): - account_id: str - email: str - display_name: str | None = None - is_active: bool - memberships: list[dict[str, Any]] = Field(default_factory=list) - roles: list[RoleSummary] = Field(default_factory=list) - last_login_at: datetime | None = None - - -class SystemAccountListResponse(BaseModel): - accounts: list[SystemAccountItem] - roles: list[RoleSummary] - - -class SystemAccountUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - display_name: str | None = Field(default=None, max_length=255) - is_active: bool | None = None - role_ids: list[str] | None = None - - -class SystemAccountRolesUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - role_ids: list[str] = Field(default_factory=list) - - -class SystemMembershipUpdate(BaseModel): - model_config = ConfigDict(extra="forbid") - - tenant_id: str - is_active: bool = True - role_ids: list[str] = Field(default_factory=list) - group_ids: list[str] = Field(default_factory=list) - - -class SystemAccountMembershipsUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - memberships: list[SystemMembershipUpdate] = Field(default_factory=list) - - -class SystemAccountCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - email: str - display_name: str | None = None - password: str | None = Field(default=None, min_length=10) - password_reset_required: bool = True - is_active: bool = True - role_ids: list[str] = Field(default_factory=list) - memberships: list[SystemMembershipUpdate] = Field(default_factory=list) - - -class SystemAccountCreateResponse(BaseModel): - account: SystemAccountItem - temporary_password: str | None = None - - -class PolicySourceStepItem(BaseModel): - scope_type: str - scope_id: str | None = None - label: str - applied_fields: list[str] = Field(default_factory=list) - policy: dict[str, Any] = Field(default_factory=dict) - - -class PrivacyRetentionPolicyItem(BaseModel): - model_config = ConfigDict(extra="forbid") - - store_raw_campaign_json: bool = True - raw_campaign_json_retention_days: int | None = Field(default=None, ge=0) - generated_eml_retention_days: int | None = Field(default=None, ge=0) - stored_report_detail_retention_days: int | None = Field(default=None, ge=0) - mock_mailbox_retention_days: int | None = Field(default=None, ge=0) - audit_detail_retention_days: int | None = Field(default=None, ge=0) - audit_detail_level: Literal["full", "redacted", "minimal"] = "full" - allow_lower_level_limits: dict[str, bool] = Field(default_factory=default_allow_lower_level_limits) - - @field_validator("allow_lower_level_limits", mode="before") - @classmethod - def _normalize_allow_lower_level_limits(cls, value: Any) -> Any: - return normalize_allow_lower_level_limits(value, fill_defaults=True) - - -class PrivacyRetentionPolicyPatchItem(BaseModel): - model_config = ConfigDict(extra="forbid") - - store_raw_campaign_json: bool | None = None - raw_campaign_json_retention_days: int | None = Field(default=None, ge=0) - generated_eml_retention_days: int | None = Field(default=None, ge=0) - stored_report_detail_retention_days: int | None = Field(default=None, ge=0) - mock_mailbox_retention_days: int | None = Field(default=None, ge=0) - audit_detail_retention_days: int | None = Field(default=None, ge=0) - audit_detail_level: Literal["full", "redacted", "minimal"] | None = None - allow_lower_level_limits: dict[str, bool] | None = None - - @field_validator("allow_lower_level_limits", mode="before") - @classmethod - def _normalize_allow_lower_level_limits(cls, value: Any) -> Any: - return normalize_allow_lower_level_limits(value, fill_defaults=False) - - -class PrivacyRetentionPolicyScopeRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - policy: PrivacyRetentionPolicyPatchItem = Field(default_factory=PrivacyRetentionPolicyPatchItem) - - -class PrivacyRetentionPolicyScopeResponse(BaseModel): - scope_type: Literal["system", "tenant", "user", "group", "campaign"] - scope_id: str | None = None - policy: dict[str, Any] - effective_policy: PrivacyRetentionPolicyItem - parent_policy: PrivacyRetentionPolicyItem | None = None - effective_policy_sources: list[PolicySourceStepItem] = Field(default_factory=list) - parent_policy_sources: list[PolicySourceStepItem] = Field(default_factory=list) - - -class RetentionRunRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - dry_run: bool = True - - -class RetentionRunResponse(BaseModel): - result: dict[str, Any] - - -class SystemSettingsItem(BaseModel): - default_locale: str = "en" - allow_tenant_custom_groups: bool = True - allow_tenant_custom_roles: bool = True - allow_tenant_api_keys: bool = True - privacy_retention_policy: PrivacyRetentionPolicyItem = Field(default_factory=PrivacyRetentionPolicyItem) - settings: dict[str, Any] = Field(default_factory=dict) - - -class SystemSettingsUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - default_locale: str = Field(min_length=1, max_length=20) - allow_tenant_custom_groups: bool - allow_tenant_custom_roles: bool - allow_tenant_api_keys: bool - privacy_retention_policy: PrivacyRetentionPolicyItem | None = None - - -class GovernanceAssignment(BaseModel): - tenant_id: str - mode: Literal["available", "required"] = "available" - - -class GovernanceTemplateItem(BaseModel): - id: str - kind: Literal["group", "role"] - slug: str - name: str - description: str | None = None - permissions: list[str] = Field(default_factory=list) - effective_permission_count: int = 0 - is_active: bool = True - assignments: list[GovernanceAssignment] = Field(default_factory=list) - created_at: datetime - updated_at: datetime - - -class GovernanceTemplateListResponse(BaseModel): - templates: list[GovernanceTemplateItem] - - -class GovernanceTemplateCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - kind: Literal["group", "role"] - slug: str - name: str = Field(min_length=1, max_length=255) - description: str | None = None - permissions: list[str] = Field(default_factory=list) - is_active: bool = True - assignments: list[GovernanceAssignment] = Field(default_factory=list) - - -class GovernanceTemplateUpdateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str = Field(min_length=1, max_length=255) - description: str | None = None - permissions: list[str] = Field(default_factory=list) - is_active: bool = True - assignments: list[GovernanceAssignment] = Field(default_factory=list) - - -class ApiKeyAdminItem(BaseModel): - id: str - user_id: str - user_email: str - name: str - prefix: str - scopes: list[str] = Field(default_factory=list) - expires_at: datetime | None = None - last_used_at: datetime | None = None - revoked_at: datetime | None = None - created_at: datetime - - -class ApiKeyListResponse(BaseModel): - api_keys: list[ApiKeyAdminItem] - - -class AdminApiKeyCreateRequest(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str = Field(min_length=1, max_length=255) - user_id: str | None = None - scopes: list[str] = Field(default_factory=list) - expires_at: datetime | None = None - - -class AdminApiKeyCreateResponse(ApiKeyAdminItem): - secret: str - - -class AuditAdminItem(BaseModel): - id: str - scope: Literal["tenant", "system"] = "tenant" - tenant_id: str | None = None - actor_email: str | None = None - action: str - object_type: str | None = None - object_id: str | None = None - details: dict[str, Any] = Field(default_factory=dict) - created_at: datetime - - -class AuditAdminListResponse(BaseModel): - items: list[AuditAdminItem] - total: int - page: int = 1 - page_size: int = 100 - pages: int = 1 diff --git a/src/govoplan_core/api/v1/audit.py b/src/govoplan_core/api/v1/audit.py deleted file mode 100644 index 63eee57..0000000 --- a/src/govoplan_core/api/v1/audit.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -from fastapi import APIRouter, Depends, Query -from sqlalchemy.orm import Session - -from govoplan_core.api.v1.schemas import AuditLogItemResponse, AuditLogListResponse -from govoplan_core.auth.dependencies import ApiPrincipal, require_scope -from govoplan_core.db.models import AuditLog -from govoplan_core.db.session import get_session - -router = APIRouter(prefix="/audit", tags=["audit"]) - - -@router.get("", response_model=AuditLogListResponse) -def list_audit_log( - limit: int = Query(default=100, ge=1, le=500), - offset: int = Query(default=0, ge=0), - action: str | None = None, - object_type: str | None = None, - object_id: str | None = None, - session: Session = Depends(get_session), - principal: ApiPrincipal = Depends(require_scope("audit:read")), -): - query = session.query(AuditLog).filter(AuditLog.tenant_id == principal.tenant_id) - if action: - query = query.filter(AuditLog.action == action) - if object_type: - query = query.filter(AuditLog.object_type == object_type) - if object_id: - query = query.filter(AuditLog.object_id == object_id) - items = query.order_by(AuditLog.created_at.desc()).offset(offset).limit(limit).all() - return AuditLogListResponse(items=[AuditLogItemResponse.model_validate(item) for item in items]) diff --git a/src/govoplan_core/api/v1/auth.py b/src/govoplan_core/api/v1/auth.py deleted file mode 100644 index 3f35b37..0000000 --- a/src/govoplan_core/api/v1/auth.py +++ /dev/null @@ -1,293 +0,0 @@ -from __future__ import annotations - -from fastapi import APIRouter, Depends, HTTPException, Request, Response, status -from sqlalchemy.orm import Session - -from govoplan_core.api.v1.schemas import ( - GroupInfo, - LoginRequest, - LoginResponse, - MeResponse, - ProfileUpdateRequest, - RoleInfo, - SwitchTenantRequest, - TenantInfo, - TenantMembershipInfo, - UserInfo, -) -from govoplan_core.auth.dependencies import ApiPrincipal, get_api_principal -from govoplan_core.audit.logging import audit_from_principal -from govoplan_core.db.models import Account, Tenant, User -from govoplan_core.db.session import get_session -from govoplan_core.security.passwords import verify_password -from govoplan_core.security.permissions import normalize_email -from govoplan_core.settings import settings -from govoplan_core.security.sessions import ( - collect_system_roles, - collect_tenant_memberships, - collect_user_groups, - collect_user_roles, - collect_user_scopes, - create_auth_session, - switch_auth_session_tenant, -) -from govoplan_core.security.time import utc_now - -router = APIRouter(prefix="/auth", tags=["auth"]) - - -def _cookie_samesite() -> str: - value = settings.auth_cookie_samesite.lower().strip() - if value not in {"lax", "strict", "none"}: - return "lax" - if value == "none" and not settings.auth_cookie_secure: - return "lax" - return value - - -def _set_auth_cookies(response: Response, created) -> None: - max_age = max(0, int((created.model.expires_at - utc_now()).total_seconds())) - common = { - "secure": settings.auth_cookie_secure, - "samesite": _cookie_samesite(), - "max_age": max_age, - "path": "/", - } - if settings.auth_cookie_domain: - common["domain"] = settings.auth_cookie_domain - response.set_cookie(settings.auth_session_cookie_name, created.token, httponly=True, **common) - response.set_cookie(settings.auth_csrf_cookie_name, created.csrf_token, httponly=False, **common) - - -def _clear_auth_cookies(response: Response) -> None: - kwargs = {"path": "/"} - if settings.auth_cookie_domain: - kwargs["domain"] = settings.auth_cookie_domain - response.delete_cookie(settings.auth_session_cookie_name, **kwargs) - response.delete_cookie(settings.auth_csrf_cookie_name, **kwargs) - - -def _tenant_info(tenant: Tenant) -> TenantInfo: - return TenantInfo( - id=tenant.id, - slug=tenant.slug, - name=tenant.name, - is_active=tenant.is_active, - default_locale=tenant.default_locale, - ) - - -def _user_info(user: User, account: Account) -> UserInfo: - return UserInfo( - id=user.id, - account_id=account.id, - email=account.email, - display_name=account.display_name or user.display_name, - tenant_display_name=user.display_name, - is_tenant_admin=user.is_tenant_admin, - password_reset_required=account.password_reset_required, - ) - - -def _roles_info(roles, *, level: str = "tenant") -> list[RoleInfo]: - return [ - RoleInfo( - id=role.id, - slug=role.slug, - name=role.name, - permissions=role.permissions or [], - level=level, - ) - for role in roles - ] - - -def _groups_info(groups) -> list[GroupInfo]: - return [GroupInfo(id=group.id, slug=group.slug, name=group.name) for group in groups] - - -def _tenant_memberships(session: Session, account: Account) -> list[TenantMembershipInfo]: - memberships: list[TenantMembershipInfo] = [] - for user, tenant in collect_tenant_memberships(session, account): - roles = collect_user_roles(session, user) - memberships.append( - TenantMembershipInfo( - id=tenant.id, - slug=tenant.slug, - name=tenant.name, - is_active=tenant.is_active and user.is_active, - default_locale=tenant.default_locale, - roles=[role.slug for role in roles], - ) - ) - return memberships - - -def _resolve_login_user(session: Session, payload: LoginRequest) -> tuple[Account, User, Tenant]: - account = ( - session.query(Account) - .filter(Account.normalized_email == normalize_email(payload.email), Account.is_active.is_(True)) - .one_or_none() - ) - if account is None or not verify_password(payload.password, account.password_hash): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid login") - - query = ( - session.query(User, Tenant) - .join(Tenant, Tenant.id == User.tenant_id) - .filter( - User.account_id == account.id, - User.is_active.is_(True), - Tenant.is_active.is_(True), - ) - ) - if payload.tenant_slug: - query = query.filter(Tenant.slug == payload.tenant_slug) - row = query.order_by(Tenant.name.asc()).first() - if row is None: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="No active tenant membership") - return account, row[0], row[1] - - -def _me_response( - session: Session, - *, - account: Account, - user: User, - tenant: Tenant, - effective_scopes: list[str] | None = None, - include_system: bool = True, - include_all_memberships: bool = True, -) -> MeResponse: - tenant_roles = collect_user_roles(session, user) - system_roles = collect_system_roles(session, account) if include_system else [] - groups = collect_user_groups(session, user) - active_tenant = _tenant_info(tenant) - memberships = _tenant_memberships(session, account) if include_all_memberships else [ - TenantMembershipInfo( - id=tenant.id, - slug=tenant.slug, - name=tenant.name, - is_active=tenant.is_active and user.is_active, - default_locale=tenant.default_locale, - roles=[role.slug for role in tenant_roles], - ) - ] - return MeResponse( - user=_user_info(user, account), - tenant=active_tenant, - active_tenant=active_tenant, - tenants=memberships, - scopes=effective_scopes if effective_scopes is not None else collect_user_scopes(session, user, include_system=include_system), - roles=_roles_info(tenant_roles) + _roles_info(system_roles, level="system"), - groups=_groups_info(groups), - ) - - -@router.post("/login", response_model=LoginResponse) -def login(payload: LoginRequest, request: Request, response: Response, session: Session = Depends(get_session)): - account, user, tenant = _resolve_login_user(session, payload) - user_agent = request.headers.get("user-agent") - ip_address = request.client.host if request.client else None - created = create_auth_session( - session, - user=user, - hours=settings.auth_session_hours, - user_agent=user_agent, - ip_address=ip_address, - ) - me_payload = _me_response(session, account=account, user=user, tenant=tenant) - session.commit() - _set_auth_cookies(response, created) - return LoginResponse( - access_token=created.token, - expires_at=created.model.expires_at, - **me_payload.model_dump(), - ) - - -@router.get("/me", response_model=MeResponse) -def me(principal: ApiPrincipal = Depends(get_api_principal), session: Session = Depends(get_session)): - tenant = session.get(Tenant, principal.tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Active tenant not found") - return _me_response( - session, - account=principal.account, - user=principal.user, - tenant=tenant, - effective_scopes=principal.scopes, - include_system=principal.auth_session is not None, - include_all_memberships=principal.auth_session is not None, - ) - - -@router.patch("/profile", response_model=MeResponse) -def update_profile( - payload: ProfileUpdateRequest, - principal: ApiPrincipal = Depends(get_api_principal), - session: Session = Depends(get_session), -): - if principal.auth_session is None: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="API keys cannot edit an interactive user profile") - - if "display_name" in payload.model_fields_set: - principal.account.display_name = payload.display_name.strip() if payload.display_name else None - session.add(principal.account) - if "tenant_display_name" in payload.model_fields_set: - principal.user.display_name = payload.tenant_display_name.strip() if payload.tenant_display_name else None - session.add(principal.user) - - audit_from_principal( - session, - principal, - action="profile.updated", - object_type="account", - object_id=principal.account.id, - details={"fields": sorted(payload.model_fields_set)}, - ) - tenant = session.get(Tenant, principal.tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Active tenant not found") - session.commit() - return _me_response( - session, - account=principal.account, - user=principal.user, - tenant=tenant, - include_system=True, - include_all_memberships=True, - ) - - -@router.post("/switch-tenant", response_model=MeResponse) -def switch_tenant( - payload: SwitchTenantRequest, - principal: ApiPrincipal = Depends(get_api_principal), - session: Session = Depends(get_session), -): - if principal.auth_session is None: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="API keys cannot switch tenant context") - try: - membership = switch_auth_session_tenant(session, principal.auth_session, payload.tenant_id) - except LookupError as exc: - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc - tenant = session.get(Tenant, membership.tenant_id) - if tenant is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tenant not found") - session.commit() - return _me_response(session, account=principal.account, user=membership, tenant=tenant) - - -@router.post("/logout") -def logout( - response: Response, - principal: ApiPrincipal = Depends(get_api_principal), - session: Session = Depends(get_session), -): - if principal.auth_session is not None: - principal.auth_session.revoked_at = utc_now() - session.add(principal.auth_session) - session.commit() - _clear_auth_cookies(response) - return {"ok": True} diff --git a/src/govoplan_core/api/v1/system.py b/src/govoplan_core/api/v1/system.py deleted file mode 100644 index fd81e4a..0000000 --- a/src/govoplan_core/api/v1/system.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from fastapi import APIRouter, Depends, HTTPException, status - -from govoplan_core.auth.dependencies import ApiPrincipal, require_any_scope -from govoplan_core.core.optional import module_not_found_for - -router = APIRouter(prefix="/schemas", tags=["schemas"]) - - -def _load_campaign_schema() -> dict[str, Any]: - try: - from govoplan_campaign.backend.campaign.loader import load_campaign_schema - except ModuleNotFoundError as exc: - if module_not_found_for(exc, "govoplan_campaign"): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign module is not installed") from exc - raise - return load_campaign_schema() - - -def _load_campaign_schema_ui() -> dict[str, Any]: - try: - from govoplan_campaign.backend.campaign.loader import load_campaign_schema_ui - except ModuleNotFoundError as exc: - if module_not_found_for(exc, "govoplan_campaign"): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign module is not installed") from exc - raise - return load_campaign_schema_ui() - - -@router.get("/campaign") -def get_campaign_schema( - principal: ApiPrincipal = Depends(require_any_scope("campaigns:campaign:read", "campaigns:campaign:create", "admin:roles:read")), -) -> dict[str, Any]: - """Return the authoritative campaign JSON Schema used by the backend.""" - - del principal - return _load_campaign_schema() - - -@router.get("/campaign/ui") -def get_campaign_schema_ui( - principal: ApiPrincipal = Depends(require_any_scope("campaigns:campaign:read", "campaigns:campaign:create", "admin:roles:read")), -) -> dict[str, Any]: - """Return UI metadata paired with the campaign JSON Schema.""" - - del principal - return _load_campaign_schema_ui() - diff --git a/src/govoplan_core/audit/logging.py b/src/govoplan_core/audit/logging.py index 54e3850..a5bc719 100644 --- a/src/govoplan_core/audit/logging.py +++ b/src/govoplan_core/audit/logging.py @@ -4,9 +4,9 @@ from typing import Any from sqlalchemy.orm import Session -from govoplan_core.auth.dependencies import ApiPrincipal -from govoplan_core.db.models import AuditLog +from govoplan_access.backend.auth.dependencies import ApiPrincipal from govoplan_core.privacy.retention import sanitize_audit_details_for_policy +from govoplan_audit.backend.db.models import AuditLog SENSITIVE_DETAIL_KEYS = { diff --git a/src/govoplan_core/auth/dependencies.py b/src/govoplan_core/auth/dependencies.py deleted file mode 100644 index 4cc9d51..0000000 --- a/src/govoplan_core/auth/dependencies.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from fastapi import Depends, Header, HTTPException, Request, status -from sqlalchemy.orm import Session - -from govoplan_core.db.models import Account, ApiKey, AuthSession, Tenant, User -from govoplan_core.db.session import get_session -from govoplan_core.security.module_permissions import scopes_grant_compatible -from govoplan_core.access.auth.principals import Principal -from govoplan_core.security.api_keys import authenticate_api_key -from govoplan_core.security.permissions import intersect_api_key_scopes, scopes_grant -from govoplan_core.security.sessions import authenticate_session_token, collect_user_groups, collect_user_scopes, verify_auth_session_csrf -from govoplan_core.settings import settings - - -@dataclass(slots=True) -class ApiPrincipal: - """Compatibility wrapper around the platform Principal DTO. - - Existing routers still expect ORM ``account`` and ``user`` objects. New - platform/module code should use the stable ID fields or - ``to_platform_principal()`` instead. - """ - - principal: Principal - account: Account - user: User - api_key: ApiKey | None = None - auth_session: AuthSession | None = None - - @property - def account_id(self) -> str: - return self.principal.account_id - - @property - def membership_id(self) -> str | None: - return self.principal.membership_id - - @property - def tenant_id(self) -> str: - if self.principal.tenant_id is None: - raise RuntimeError("Tenant principal has no active tenant id.") - return self.principal.tenant_id - - @property - def scopes(self) -> frozenset[str]: - return self.principal.scopes - - @property - def group_ids(self) -> frozenset[str]: - return self.principal.group_ids - - @property - def auth_method(self) -> str: - return self.principal.auth_method - - @property - def api_key_id(self) -> str | None: - return self.principal.api_key_id - - @property - def session_id(self) -> str | None: - return self.principal.session_id - - @property - def email(self) -> str | None: - return self.principal.email - - @property - def display_name(self) -> str | None: - return self.principal.display_name - - def has(self, required_scope: str) -> bool: - # Keep legacy scope aliases alive while current routers still use the - # pre-platform permission catalogue. - return scopes_grant_compatible(self.scopes, required_scope) - - def to_platform_principal(self) -> Principal: - return self.principal - - -def _extract_token(request: Request, authorization: str | None, x_api_key: str | None) -> tuple[str | None, str]: - if x_api_key: - return x_api_key.strip(), "api_key" - if authorization and authorization.lower().startswith("bearer "): - return authorization[7:].strip(), "bearer" - cookie_token = request.cookies.get(settings.auth_session_cookie_name) - if cookie_token: - return cookie_token.strip(), "cookie" - return None, "none" - - -def _requires_csrf(request: Request) -> bool: - return request.method.upper() not in {"GET", "HEAD", "OPTIONS", "TRACE"} - - -def _principal_group_ids(session: Session, user: User) -> frozenset[str]: - return frozenset(group.id for group in collect_user_groups(session, user)) - - -def _build_api_principal( - session: Session, - *, - account: Account, - user: User, - tenant_id: str, - scopes: list[str], - auth_method: str, - api_key: ApiKey | None = None, - auth_session: AuthSession | None = None, -) -> ApiPrincipal: - principal = Principal( - account_id=account.id, - membership_id=user.id, - tenant_id=tenant_id, - scopes=frozenset(scopes), - group_ids=_principal_group_ids(session, user), - auth_method=auth_method, # type: ignore[arg-type] - api_key_id=api_key.id if api_key else None, - session_id=auth_session.id if auth_session else None, - email=account.email, - display_name=account.display_name or user.display_name, - ) - return ApiPrincipal( - principal=principal, - account=account, - user=user, - api_key=api_key, - auth_session=auth_session, - ) - - -def get_api_principal( - request: Request, - session: Session = Depends(get_session), - authorization: str | None = Header(default=None), - x_api_key: str | None = Header(default=None, alias="X-API-Key"), -) -> ApiPrincipal: - token, source = _extract_token(request, authorization, x_api_key) - if not token: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Missing API key or session token") - - # API keys remain supported for CLI/automation. Their permissions are the - # intersection of the key grant and the owner's current tenant roles. - api_key = authenticate_api_key(session, token) - if api_key: - user = session.get(User, api_key.user_id) - account = session.get(Account, user.account_id) if user else None - tenant = session.get(Tenant, api_key.tenant_id) - if ( - not user or not account or not tenant - or not user.is_active or not account.is_active or not tenant.is_active - or user.tenant_id != api_key.tenant_id - ): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Inactive or inconsistent API-key principal") - user_scopes = collect_user_scopes(session, user, include_system=False) - effective_scopes = intersect_api_key_scopes(user_scopes, api_key.scopes or []) - session.commit() - return _build_api_principal( - session, - api_key=api_key, - account=account, - user=user, - tenant_id=api_key.tenant_id, - scopes=effective_scopes, - auth_method="api_key", - ) - - auth_session = authenticate_session_token(session, token) - if auth_session: - user = session.get(User, auth_session.user_id) - account = session.get(Account, auth_session.account_id) - tenant = session.get(Tenant, auth_session.tenant_id) - if ( - not user or not account or not tenant - or not user.is_active or not account.is_active or not tenant.is_active - or user.account_id != account.id - or user.tenant_id != tenant.id - ): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Inactive or inconsistent session principal") - if source == "cookie" and _requires_csrf(request): - header_token = request.headers.get("x-csrf-token") - cookie_token = request.cookies.get(settings.auth_csrf_cookie_name) - if not header_token or not cookie_token or header_token != cookie_token or not verify_auth_session_csrf(auth_session, header_token): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid or missing CSRF token") - scopes = collect_user_scopes(session, user, include_system=True) - session.commit() - return _build_api_principal( - session, - auth_session=auth_session, - account=account, - user=user, - tenant_id=user.tenant_id, - scopes=scopes, - auth_method="session", - ) - - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key or session token") - - -def has_scope(principal: ApiPrincipal, required_scope: str) -> bool: - return principal.has(required_scope) - - -def require_scope(required_scope: str): - def dependency(principal: ApiPrincipal = Depends(get_api_principal)) -> ApiPrincipal: - if not has_scope(principal, required_scope): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Missing scope: {required_scope}") - return principal - - return dependency - - -def require_any_scope(*required_scopes: str): - def dependency(principal: ApiPrincipal = Depends(get_api_principal)) -> ApiPrincipal: - if not any(has_scope(principal, required) for required in required_scopes): - joined = ", ".join(required_scopes) - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"Requires one of: {joined}") - return principal - - return dependency diff --git a/src/govoplan_core/celery_app.py b/src/govoplan_core/celery_app.py index 8886fbc..ad29c60 100644 --- a/src/govoplan_core/celery_app.py +++ b/src/govoplan_core/celery_app.py @@ -2,8 +2,13 @@ from __future__ import annotations from celery import Celery +from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, CampaignDeliveryTaskProvider +from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids +from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.runtime import configure_runtime from govoplan_core.settings import settings from govoplan_core.db.session import configure_database +from govoplan_core.server.registry import available_module_manifests, build_platform_registry configure_database(settings.database_url) @@ -30,6 +35,21 @@ def ping(): return "pong" +def _campaign_delivery_tasks() -> CampaignDeliveryTaskProvider: + raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules) + candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules) + available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True) + enabled_modules = load_startup_enabled_modules(settings.enabled_modules, available=available_modules) + registry = build_platform_registry(enabled_modules) + context = ModuleContext(registry=registry, settings=settings) + configure_runtime(context) + registry.configure_capability_context(context) + capability = registry.require_capability(CAPABILITY_CAMPAIGNS_DELIVERY_TASKS) + if not isinstance(capability, CampaignDeliveryTaskProvider): + raise RuntimeError("Campaign delivery task capability is invalid") + return capability + + @celery.task(name="multimailer.send_email", bind=True, max_retries=0) def send_email(self, job_id: str): """Send one explicitly queued campaign job. @@ -40,10 +60,9 @@ def send_email(self, job_id: str): """ from govoplan_core.db.session import get_database - from govoplan_campaign.backend.sending.jobs import send_campaign_job with get_database().SessionLocal() as session: - return send_campaign_job(session, job_id=job_id, enqueue_imap_task=True).as_dict() + return dict(_campaign_delivery_tasks().send_campaign_job(session, job_id=job_id, enqueue_imap_task=True)) @celery.task(name="multimailer.append_sent", bind=True, max_retries=None) @@ -51,13 +70,11 @@ def append_sent(self, job_id: str): """Append the exact sent MIME to the configured IMAP Sent folder.""" from govoplan_core.db.session import get_database - from govoplan_mail.backend.sending.imap import ImapAppendError - from govoplan_campaign.backend.sending.jobs import append_sent_for_job with get_database().SessionLocal() as session: try: - return append_sent_for_job(session, job_id=job_id).as_dict() - except ImapAppendError as exc: + return dict(_campaign_delivery_tasks().append_sent_for_job(session, job_id=job_id)) + except Exception as exc: if getattr(exc, "temporary", None) is True: raise self.retry(exc=exc, countdown=300) raise diff --git a/src/govoplan_core/commands/init_db.py b/src/govoplan_core/commands/init_db.py index 19a379a..60b9f5b 100644 --- a/src/govoplan_core/commands/init_db.py +++ b/src/govoplan_core/commands/init_db.py @@ -10,12 +10,13 @@ from govoplan_core.settings import settings def main() -> None: parser = argparse.ArgumentParser(description="Initialize the GovOPlaN database") + parser.add_argument("--database-url", default=settings.database_url, help="Database URL to migrate") parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key") parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create") args = parser.parse_args() - configure_database(settings.database_url) - migration = migrate_database() + configure_database(args.database_url) + migration = migrate_database(database_url=args.database_url) if migration.reconciled_revision: print(f"Reconciled legacy database marker to {migration.reconciled_revision}.") print(f"Database schema upgraded to {migration.current_revision}.") diff --git a/src/govoplan_core/commands/module_install_plan.py b/src/govoplan_core/commands/module_install_plan.py new file mode 100644 index 0000000..4d3fe17 --- /dev/null +++ b/src/govoplan_core/commands/module_install_plan.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import argparse +import json + +from govoplan_core.core.module_management import module_install_plan_commands, saved_module_install_plan +from govoplan_core.db.session import configure_database, get_database +from govoplan_core.settings import settings + + +def main() -> None: + parser = argparse.ArgumentParser(description="Print the GovOPlaN module package install plan.") + parser.add_argument("--database-url", default=settings.database_url, help="Database URL containing system_settings.") + parser.add_argument("--format", choices=("shell", "json"), default="shell", help="Output format.") + args = parser.parse_args() + + configure_database(str(args.database_url)) + with get_database().session() as session: + plan = saved_module_install_plan(session) + + if args.format == "json": + print(json.dumps({ + "updated_at": plan.updated_at, + "items": [item.as_dict() for item in plan.items], + "commands": list(module_install_plan_commands(plan)), + }, indent=2, sort_keys=True)) + return + + if not plan.items: + print("# No planned module package changes.") + return + + print("# GovOPlaN module package install plan") + if plan.updated_at: + print(f"# Updated: {plan.updated_at}") + for command in module_install_plan_commands(plan): + print(command) + + +if __name__ == "__main__": + main() diff --git a/src/govoplan_core/commands/module_installer.py b/src/govoplan_core/commands/module_installer.py new file mode 100644 index 0000000..7fddd48 --- /dev/null +++ b/src/govoplan_core/commands/module_installer.py @@ -0,0 +1,470 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +import time +from typing import Any + +from govoplan_core.core.module_installer import ( + ModuleInstallerError, + cancel_module_installer_request, + claim_next_module_installer_request, + default_installer_runtime_dir, + list_module_installer_runs, + list_module_installer_requests, + module_installer_daemon_status, + module_install_preflight, + module_installer_lock_status, + queue_module_installer_request, + read_module_installer_run, + read_module_installer_request, + retry_module_installer_request, + rollback_module_install_run, + run_module_install_plan, + supervise_module_install_plan, + update_module_installer_request, + update_module_installer_daemon_status, +) +from govoplan_core.core.module_package_catalog import sign_module_package_catalog, validate_module_package_catalog +from govoplan_core.core.module_management import ( + configured_enabled_modules, + saved_desired_enabled_modules, + saved_module_install_plan, +) +from govoplan_core.db.session import configure_database, get_database +from govoplan_core.server.registry import available_module_manifests +from govoplan_core.settings import settings + + +def main() -> int: + parser = argparse.ArgumentParser(description="Preflight, apply, or roll back a GovOPlaN module package install plan.") + parser.add_argument("--database-url", default=settings.database_url, help="Database URL containing system_settings.") + parser.add_argument("--runtime-dir", type=Path, help="Directory for installer locks and run snapshots.") + parser.add_argument("--webui-root", type=Path, default=_default_webui_root(), help="Core WebUI root for npm package changes.") + parser.add_argument("--npm-bin", default="npm", help="npm executable to use for WebUI package changes.") + parser.add_argument("--build-webui", action="store_true", help="Run npm run build after npm install.") + parser.add_argument("--migrate", action="store_true", help="Run core/module database migrations after package changes and before restart.") + parser.add_argument("--database-backup-command", help="Shell command that creates a database backup before --migrate for non-SQLite databases.") + parser.add_argument("--database-restore-command", help="Shell command that restores the database backup during rollback.") + parser.add_argument("--database-restore-check-command", help="Shell command that validates the created backup can be used before migrations proceed.") + parser.add_argument("--no-activate-installed-modules", action="store_true", help="Do not add successfully installed modules to saved startup state.") + parser.add_argument("--keep-uninstalled-modules-in-desired", action="store_true", help="Do not remove successfully uninstalled modules from saved startup state.") + parser.add_argument("--format", choices=("shell", "json"), default="shell", help="Output format for preflight/dry-run.") + parser.add_argument("--apply", action="store_true", help="Execute the saved plan after preflight passes.") + parser.add_argument("--dry-run", action="store_true", help="Create a run record and snapshots but do not execute commands.") + parser.add_argument("--supervise", action="store_true", help="Apply the plan, run optional restart/health checks, and roll back if the server does not recover.") + parser.add_argument("--restart-command", help="Shell command run after apply and after automatic rollback, for example a systemctl restart command.") + parser.add_argument("--health-url", help="HTTP health URL to poll after apply, for example http://127.0.0.1:8000/health.") + parser.add_argument("--health-timeout-seconds", type=float, default=60.0, help="Maximum time to wait for health after restart.") + parser.add_argument("--health-interval-seconds", type=float, default=2.0, help="Delay between health probes.") + parser.add_argument("--rollback", metavar="RUN_ID", help="Restore package snapshots from a previous installer run.") + parser.add_argument("--list-runs", action="store_true", help="List recent installer run records and lock status.") + parser.add_argument("--show-run", metavar="RUN_ID", help="Print one installer run record.") + parser.add_argument("--lock-status", action="store_true", help="Print the current installer lock status.") + parser.add_argument("--list-requests", action="store_true", help="List queued/running/completed installer requests.") + parser.add_argument("--show-request", metavar="REQUEST_ID", help="Print one installer request record.") + parser.add_argument("--cancel-request", metavar="REQUEST_ID", help="Cancel a queued installer request.") + parser.add_argument("--retry-request", metavar="REQUEST_ID", help="Queue a retry for a failed or cancelled installer request.") + parser.add_argument("--enqueue-supervised", action="store_true", help="Queue a supervised installer request for a running daemon instead of applying immediately.") + parser.add_argument("--daemon-status", action="store_true", help="Print installer daemon heartbeat/status.") + parser.add_argument("--daemon", action="store_true", help="Run the installer request daemon.") + parser.add_argument("--daemon-once", action="store_true", help="Process at most one queued request and exit.") + parser.add_argument("--poll-interval-seconds", type=float, default=2.0, help="Delay between daemon queue polls.") + parser.add_argument("--validate-package-catalog", nargs="?", const="", metavar="PATH", help="Validate a module package catalog JSON file, or the configured catalog when PATH is omitted.") + parser.add_argument("--require-signed-catalog", action="store_true", help="Require package catalog validation to trust an Ed25519 signature.") + parser.add_argument("--approved-catalog-channel", action="append", default=[], help="Approved package catalog channel; may be repeated.") + parser.add_argument("--catalog-trusted-key", action="append", default=[], metavar="KEY_ID=BASE64_PUBLIC_KEY", help="Trusted Ed25519 catalog public key; may be repeated.") + parser.add_argument("--sign-package-catalog", type=Path, metavar="PATH", help="Sign a module package catalog JSON file.") + parser.add_argument("--catalog-signing-key-id", help="Key id to record when signing a package catalog.") + parser.add_argument("--catalog-signing-private-key", type=Path, help="PEM Ed25519 private key used by --sign-package-catalog.") + parser.add_argument("--catalog-output", type=Path, help="Output path for --sign-package-catalog. Defaults to overwriting the input file.") + args = parser.parse_args() + + runtime_dir = args.runtime_dir or default_installer_runtime_dir(args.database_url) + try: + if args.sign_package_catalog: + if not args.catalog_signing_key_id or not args.catalog_signing_private_key: + raise ModuleInstallerError("--sign-package-catalog requires --catalog-signing-key-id and --catalog-signing-private-key.") + path = sign_module_package_catalog( + path=args.sign_package_catalog, + key_id=args.catalog_signing_key_id, + private_key_path=args.catalog_signing_private_key, + output_path=args.catalog_output, + ) + _print_result({"signed": True, "path": str(path), "key_id": args.catalog_signing_key_id}, output_format=args.format) + return 0 + if args.validate_package_catalog is not None: + path = Path(args.validate_package_catalog).expanduser() if args.validate_package_catalog else None + result = validate_module_package_catalog( + path, + require_trusted=args.require_signed_catalog, + approved_channels=tuple(args.approved_catalog_channel), + trusted_keys=_trusted_catalog_keys_from_args(args.catalog_trusted_key), + ) + _print_result(result, output_format=args.format) + return 0 if result.get("valid") else 1 + if args.daemon_status: + _print_result(module_installer_daemon_status(runtime_dir=runtime_dir), output_format=args.format) + return 0 + if args.daemon or args.daemon_once: + return _run_daemon(args=args, runtime_dir=runtime_dir) + if args.list_requests: + _print_result({ + "requests": list(list_module_installer_requests(runtime_dir=runtime_dir)), + "daemon": module_installer_daemon_status(runtime_dir=runtime_dir), + }, output_format=args.format) + return 0 + if args.show_request: + _print_result(read_module_installer_request(runtime_dir=runtime_dir, request_id=args.show_request), output_format=args.format) + return 0 + if args.cancel_request: + _print_result(cancel_module_installer_request(runtime_dir=runtime_dir, request_id=args.cancel_request, cancelled_by="cli"), output_format=args.format) + return 0 + if args.retry_request: + _print_result(retry_module_installer_request(runtime_dir=runtime_dir, request_id=args.retry_request, requested_by="cli"), output_format=args.format) + return 0 + if args.enqueue_supervised: + request = queue_module_installer_request( + runtime_dir=runtime_dir, + requested_by="cli", + options=_request_options_from_args(args), + ) + _print_result(request, output_format=args.format) + return 0 + if args.list_runs: + _print_result({ + "runs": list(list_module_installer_runs(runtime_dir=runtime_dir)), + "lock": module_installer_lock_status(runtime_dir=runtime_dir), + }, output_format=args.format) + return 0 + if args.show_run: + _print_result(read_module_installer_run(runtime_dir=runtime_dir, run_id=args.show_run), output_format=args.format) + return 0 + if args.lock_status: + _print_result(module_installer_lock_status(runtime_dir=runtime_dir), output_format=args.format) + return 0 + if args.rollback: + result = rollback_module_install_run( + run_id=args.rollback, + runtime_dir=runtime_dir, + npm_bin=args.npm_bin, + build_webui=args.build_webui, + database_restore_command=args.database_restore_command, + database_url=str(args.database_url), + ) + _print_result(result.as_dict(), output_format=args.format) + return 0 if result.return_code == 0 else 1 + + configure_database(str(args.database_url)) + available = available_module_manifests(ignore_load_errors=True) + with get_database().session() as session: + configured = configured_enabled_modules(settings.enabled_modules) + desired = saved_desired_enabled_modules(session, configured) + plan = saved_module_install_plan(session) + if args.supervise: + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + database_url=str(args.database_url), + runtime_dir=runtime_dir, + webui_root=args.webui_root, + npm_bin=args.npm_bin, + build_webui=args.build_webui, + migrate_database=args.migrate, + database_backup_command=args.database_backup_command, + database_restore_command=args.database_restore_command, + database_restore_check_command=args.database_restore_check_command, + activate_installed_modules=not args.no_activate_installed_modules, + remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired, + restart_command=args.restart_command, + health_url=args.health_url, + health_timeout_seconds=args.health_timeout_seconds, + health_interval_seconds=args.health_interval_seconds, + ) + _print_result(result.as_dict(), output_format=args.format) + return 0 if result.return_code == 0 else 1 + if args.apply or args.dry_run: + result = run_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + database_url=str(args.database_url), + runtime_dir=runtime_dir, + webui_root=args.webui_root, + npm_bin=args.npm_bin, + build_webui=args.build_webui, + migrate_database=args.migrate, + database_backup_command=args.database_backup_command, + database_restore_command=args.database_restore_command, + database_restore_check_command=args.database_restore_check_command, + activate_installed_modules=not args.no_activate_installed_modules, + remove_uninstalled_modules_from_desired=not args.keep_uninstalled_modules_in_desired, + dry_run=args.dry_run, + ) + _print_result(result.as_dict(), output_format=args.format) + return 0 if result.return_code == 0 else 1 + + from govoplan_core.core.maintenance import saved_maintenance_mode + + maintenance = saved_maintenance_mode(session) + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + maintenance_mode=maintenance.enabled, + session=session, + webui_root=args.webui_root, + runtime_dir=runtime_dir, + ) + _print_preflight(preflight.as_dict(), output_format=args.format) + return 0 if preflight.allowed else 1 + except ModuleInstallerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +def _default_webui_root() -> Path: + return Path(__file__).resolve().parents[3] / "webui" + + +def _run_daemon(*, args: argparse.Namespace, runtime_dir: Path) -> int: + update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={ + "status": "running", + "mode": "once" if args.daemon_once else "daemon", + "started_at": _now(), + "current_request_id": None, + }, + ) + try: + while True: + update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={ + "status": "polling", + "last_poll_at": _now(), + "current_request_id": None, + }, + ) + request = claim_next_module_installer_request(runtime_dir=runtime_dir) + if request is not None: + request_id = str(request["request_id"]) + update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={ + "status": "processing", + "current_request_id": request_id, + "last_claimed_at": _now(), + }, + ) + _process_request(request=request, args=args, runtime_dir=runtime_dir) + update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={ + "status": "polling", + "current_request_id": None, + "last_completed_request_id": request_id, + }, + ) + if args.daemon_once: + return 0 + elif args.daemon_once: + return 0 + time.sleep(max(args.poll_interval_seconds, 0.2)) + finally: + update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={ + "status": "stopped", + "stopped_at": _now(), + "current_request_id": None, + }, + ) + + +def _process_request(*, request: dict[str, object], args: argparse.Namespace, runtime_dir: Path) -> None: + request_id = str(request["request_id"]) + options = request.get("options") if isinstance(request.get("options"), dict) else {} + configure_database(str(args.database_url)) + try: + available = available_module_manifests(ignore_load_errors=True) + with get_database().session() as session: + configured = configured_enabled_modules(settings.enabled_modules) + desired = saved_desired_enabled_modules(session, configured) + plan = saved_module_install_plan(session) + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=desired, + desired_enabled=desired, + database_url=str(args.database_url), + runtime_dir=runtime_dir, + webui_root=_path_option(options, "webui_root") or args.webui_root, + npm_bin=_str_option(options, "npm_bin") or args.npm_bin, + build_webui=_bool_option(options, "build_webui", args.build_webui), + migrate_database=_bool_option(options, "migrate_database", args.migrate), + database_backup_command=_str_option(options, "database_backup_command") or args.database_backup_command, + database_restore_command=_str_option(options, "database_restore_command") or args.database_restore_command, + database_restore_check_command=_str_option(options, "database_restore_check_command") or args.database_restore_check_command, + activate_installed_modules=_bool_option(options, "activate_installed_modules", not args.no_activate_installed_modules), + remove_uninstalled_modules_from_desired=_bool_option(options, "remove_uninstalled_modules_from_desired", not args.keep_uninstalled_modules_in_desired), + restart_commands=_list_option(options, "restart_commands") or _list_option(options, "restart_command") or ([args.restart_command] if args.restart_command else []), + health_urls=_list_option(options, "health_urls") or _list_option(options, "health_url") or ([args.health_url] if args.health_url else []), + health_timeout_seconds=_float_option(options, "health_timeout_seconds", args.health_timeout_seconds), + health_interval_seconds=_float_option(options, "health_interval_seconds", args.health_interval_seconds), + ) + update_module_installer_request( + runtime_dir=runtime_dir, + request_id=request_id, + patch={ + "status": "completed" if result.return_code == 0 else "failed", + "finished_at": _now(), + "result": result.as_dict(), + }, + ) + except Exception as exc: + update_module_installer_request( + runtime_dir=runtime_dir, + request_id=request_id, + patch={ + "status": "failed", + "finished_at": _now(), + "error": str(exc), + }, + ) + + +def _request_options_from_args(args: argparse.Namespace) -> dict[str, object]: + return { + "webui_root": str(args.webui_root) if args.webui_root else None, + "npm_bin": args.npm_bin, + "build_webui": args.build_webui, + "migrate_database": args.migrate, + "database_backup_command": args.database_backup_command, + "database_restore_command": args.database_restore_command, + "database_restore_check_command": args.database_restore_check_command, + "activate_installed_modules": not args.no_activate_installed_modules, + "remove_uninstalled_modules_from_desired": not args.keep_uninstalled_modules_in_desired, + "restart_commands": [args.restart_command] if args.restart_command else [], + "health_urls": [args.health_url] if args.health_url else [], + "health_timeout_seconds": args.health_timeout_seconds, + "health_interval_seconds": args.health_interval_seconds, + } + + +def _str_option(options: object, key: str) -> str | None: + if not isinstance(options, dict): + return None + value = options.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _path_option(options: object, key: str) -> Path | None: + value = _str_option(options, key) + return Path(value).expanduser() if value else None + + +def _bool_option(options: object, key: str, default: bool) -> bool: + if not isinstance(options, dict): + return default + value = options.get(key) + return value if isinstance(value, bool) else default + + +def _float_option(options: object, key: str, default: float) -> float: + if not isinstance(options, dict): + return default + value = options.get(key) + if isinstance(value, int | float): + return float(value) + return default + + +def _list_option(options: object, key: str) -> list[str]: + if not isinstance(options, dict): + return [] + value = options.get(key) + if isinstance(value, str): + return [item.strip() for item in value.splitlines() if item.strip()] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [] + + +def _trusted_catalog_keys_from_args(values: list[str]) -> dict[str, str] | None: + if not values: + return None + keys: dict[str, str] = {} + for value in values: + key_id, separator, public_key = value.partition("=") + if not separator or not key_id.strip() or not public_key.strip(): + raise ModuleInstallerError("--catalog-trusted-key must use KEY_ID=BASE64_PUBLIC_KEY.") + keys[key_id.strip()] = public_key.strip() + return keys + + +def _now() -> str: + from datetime import UTC, datetime + + return datetime.now(tz=UTC).isoformat() + + +def _print_preflight(payload: dict[str, object], *, output_format: str) -> None: + if output_format == "json": + print(json.dumps(payload, indent=2, sort_keys=True)) + return + print(f"# Allowed: {payload['allowed']}") + print(f"# Maintenance mode: {payload['maintenance_mode']}") + print(f"# Frontend rebuild required: {payload['frontend_rebuild_required']}") + for issue in payload.get("issues", []): + if not isinstance(issue, dict): + continue + print(f"# {issue.get('severity')}: {issue.get('code')}: {issue.get('message')}") + commands = payload.get("commands") + if isinstance(commands, list) and commands: + print("") + for command in commands: + print(command) + checklist = payload.get("checklist") + if isinstance(checklist, list) and checklist: + print("") + for item in checklist: + if not isinstance(item, dict): + continue + detail = f": {item.get('detail')}" if item.get("detail") else "" + print(f"# checklist {item.get('status')}: {item.get('label')}{detail}") + + +def _print_result(payload: dict[str, object], *, output_format: str) -> None: + if output_format == "json": + print(json.dumps(payload, indent=2, sort_keys=True)) + return + if "runs" in payload or "requests" in payload or "locked" in payload or "request_id" in payload or "run_id" not in payload: + print(json.dumps(payload, indent=2, sort_keys=True)) + return + print(f"# Run: {payload['run_id']}") + print(f"# Status: {payload['status']}") + print(f"# Record: {payload['record_path']}") + if payload.get("error"): + print(f"# Error: {payload['error']}") + commands = payload.get("commands") + if isinstance(commands, list) and commands: + print("") + for command in commands: + print(command) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/govoplan_core/commands/module_verify.py b/src/govoplan_core/commands/module_verify.py new file mode 100644 index 0000000..2648a51 --- /dev/null +++ b/src/govoplan_core/commands/module_verify.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import argparse +import sys + +from govoplan_core.core.module_installer import module_manifest_compatibility_issues +from govoplan_core.server.registry import available_module_manifests + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify installed GovOPlaN module manifests in a fresh interpreter.") + parser.add_argument("--installed", action="append", default=[], help="Module id expected to be discoverable.") + parser.add_argument("--absent", action="append", default=[], help="Module id expected not to be discoverable.") + args = parser.parse_args() + + manifests = available_module_manifests(ignore_load_errors=True) + failed = False + for module_id in args.installed: + if module_id not in manifests: + print(f"missing expected module manifest: {module_id}", file=sys.stderr) + failed = True + for module_id in args.absent: + if module_id in manifests: + print(f"unexpected module manifest still discoverable: {module_id}", file=sys.stderr) + failed = True + for issue in module_manifest_compatibility_issues(manifests): + print(f"{issue.severity}: {issue.module_id or 'global'}: {issue.message}", file=sys.stderr) + if issue.severity == "blocker": + failed = True + if failed: + return 1 + print("Module manifests verified.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/govoplan_core/core/access.py b/src/govoplan_core/core/access.py new file mode 100644 index 0000000..df7fe5e --- /dev/null +++ b/src/govoplan_core/core/access.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import datetime +from typing import Literal, Protocol, runtime_checkable + +from govoplan_core.core.modules import AccessDecision + + +ACCESS_MODULE_ID = "access" +TENANCY_MODULE_ID = "tenancy" +POLICY_MODULE_ID = "policy" +AUDIT_MODULE_ID = "audit" + +CAPABILITY_ACCESS_PRINCIPAL_RESOLVER = "access.principalResolver" +CAPABILITY_ACCESS_DIRECTORY = "access.directory" +CAPABILITY_ACCESS_PERMISSION_EVALUATOR = "access.permissionEvaluator" +CAPABILITY_ACCESS_RESOURCE_ACCESS = "access.resourceAccess" +CAPABILITY_ACCESS_TENANT_PROVISIONER = "access.tenantProvisioner" +CAPABILITY_ACCESS_ADMINISTRATION = "access.administration" +CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER = "access.governanceMaterializer" +CAPABILITY_TENANCY_TENANT_RESOLVER = "tenancy.tenantResolver" +CAPABILITY_SECURITY_SECRET_PROVIDER = "security.secretProvider" +CAPABILITY_AUDIT_SINK = "audit.sink" + +ACCESS_CAPABILITY_NAMES = frozenset( + { + CAPABILITY_ACCESS_PRINCIPAL_RESOLVER, + CAPABILITY_ACCESS_DIRECTORY, + CAPABILITY_ACCESS_PERMISSION_EVALUATOR, + CAPABILITY_ACCESS_RESOURCE_ACCESS, + CAPABILITY_ACCESS_TENANT_PROVISIONER, + CAPABILITY_ACCESS_ADMINISTRATION, + CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER, + CAPABILITY_TENANCY_TENANT_RESOLVER, + CAPABILITY_SECURITY_SECRET_PROVIDER, + CAPABILITY_AUDIT_SINK, + } +) + +AuthMethod = Literal["session", "api_key", "service_account"] +AccessSubjectKind = Literal["account", "user", "group", "role", "tenant", "api_key", "service_account"] +AccessStatus = Literal["active", "inactive", "suspended"] +PermissionLevel = Literal["system", "tenant"] +AuditOutcome = Literal["success", "failure", "denied", "unknown"] + + +@dataclass(frozen=True, slots=True) +class PrincipalRef: + account_id: str + membership_id: str | None + tenant_id: str | None + scopes: frozenset[str] = field(default_factory=frozenset) + group_ids: frozenset[str] = field(default_factory=frozenset) + auth_method: AuthMethod = "session" + api_key_id: str | None = None + session_id: str | None = None + service_account_id: str | None = None + email: str | None = None + display_name: str | None = None + + +@dataclass(frozen=True, slots=True) +class AccountRef: + id: str + email: str + display_name: str | None = None + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class TenantRef: + id: str + name: str + slug: str | None = None + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class UserRef: + id: str + account_id: str + tenant_id: str + email: str | None = None + display_name: str | None = None + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class GroupRef: + id: str + tenant_id: str + name: str + status: AccessStatus = "active" + + +@dataclass(frozen=True, slots=True) +class RoleRef: + id: str + name: str + level: PermissionLevel + tenant_id: str | None = None + managed: bool = False + protected: bool = False + + +@dataclass(frozen=True, slots=True) +class AccessSubjectRef: + kind: AccessSubjectKind + id: str + tenant_id: str | None = None + label: str | None = None + + +@dataclass(frozen=True, slots=True) +class TenantOwnerCandidateRef: + account_id: str + email: str + display_name: str | None = None + + +@dataclass(frozen=True, slots=True) +class GovernanceTemplateMaterialization: + template_id: str + kind: Literal["group", "role"] + tenant_id: str + slug: str + name: str + description: str | None = None + permissions: tuple[str, ...] = () + is_active: bool = True + required: bool = False + + +@dataclass(frozen=True, slots=True) +class AuditEvent: + event_type: str + action: str + outcome: AuditOutcome = "unknown" + actor: PrincipalRef | None = None + tenant_id: str | None = None + resource_type: str | None = None + resource_id: str | None = None + occurred_at: datetime | None = None + details: Mapping[str, object] = field(default_factory=dict) + + +@runtime_checkable +class PrincipalResolver(Protocol): + def resolve_request(self, request: object, *, session: object | None = None) -> PrincipalRef: + ... + + +@runtime_checkable +class TenantResolver(Protocol): + def get_tenant(self, tenant_id: str) -> TenantRef | None: + ... + + def current_tenant(self, principal: PrincipalRef) -> TenantRef | None: + ... + + +@runtime_checkable +class AccessDirectory(Protocol): + def get_account(self, account_id: str) -> AccountRef | None: + ... + + def get_user(self, user_id: str) -> UserRef | None: + ... + + def get_users(self, user_ids: Iterable[str]) -> Mapping[str, UserRef]: + ... + + def users_for_tenant(self, tenant_id: str) -> Sequence[UserRef]: + ... + + def get_group(self, group_id: str) -> GroupRef | None: + ... + + def get_groups(self, group_ids: Iterable[str]) -> Mapping[str, GroupRef]: + ... + + def groups_for_tenant(self, tenant_id: str) -> Sequence[GroupRef]: + ... + + def groups_for_user(self, user_id: str, *, tenant_id: str) -> Sequence[GroupRef]: + ... + + def display_label(self, subject: AccessSubjectRef) -> str | None: + ... + + +@runtime_checkable +class PermissionEvaluator(Protocol): + def scopes_grant(self, scopes: Iterable[str], required_scope: str) -> bool: + ... + + def has_scope(self, principal: PrincipalRef, required_scope: str) -> bool: + ... + + def explain_scope(self, principal: PrincipalRef, required_scope: str) -> AccessDecision: + ... + + +@runtime_checkable +class ResourceAccessService(Protocol): + def explain(self, principal: PrincipalRef, *, resource_type: str, resource_id: str, action: str) -> AccessDecision: + ... + + +@runtime_checkable +class TenantAccessProvisioner(Protocol): + def ensure_default_roles(self, session: object, tenant: object | None = None) -> Mapping[str, object]: + ... + + def tenant_owner_candidates(self, session: object) -> Sequence[TenantOwnerCandidateRef]: + ... + + def ensure_tenant_owner_membership( + self, + session: object, + *, + tenant: object, + owner_account_id: str, + ) -> str: + ... + + +@runtime_checkable +class AccessAdministration(Protocol): + def system_account_count(self, session: object) -> int: + ... + + def role_count_for_tenant(self, session: object, tenant_id: str) -> int: + ... + + def active_api_key_count_for_tenant(self, session: object, tenant_id: str) -> int: + ... + + def actor_email_by_user_id(self, session: object, user_ids: Iterable[str]) -> Mapping[str, str | None]: + ... + + def user_ids_for_actor_filter(self, session: object, *, operator: str, value: str) -> Sequence[str]: + ... + + +@runtime_checkable +class AccessGovernanceMaterializer(Protocol): + def sync_template(self, session: object, template: GovernanceTemplateMaterialization) -> None: + ... + + def remove_template(self, session: object, template: GovernanceTemplateMaterialization) -> None: + ... + + +@runtime_checkable +class SecretProvider(Protocol): + def store_secret(self, *, scope: str, name: str, value: str) -> str: + ... + + def read_secret(self, secret_ref: str) -> str | None: + ... + + def delete_secret(self, secret_ref: str) -> None: + ... + + +@runtime_checkable +class AuditSink(Protocol): + def record(self, event: AuditEvent) -> None: + ... diff --git a/src/govoplan_core/core/campaigns.py b/src/govoplan_core/core/campaigns.py new file mode 100644 index 0000000..de2927d --- /dev/null +++ b/src/govoplan_core/core/campaigns.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime +from typing import Protocol, runtime_checkable + + +CAPABILITY_CAMPAIGNS_MAIL_POLICY_CONTEXT = "campaigns.mailPolicyContext" +CAPABILITY_CAMPAIGNS_ACCESS = "campaigns.access" +CAPABILITY_CAMPAIGNS_POLICY_CONTEXT = "campaigns.policyContext" +CAPABILITY_CAMPAIGNS_DELIVERY_TASKS = "campaigns.deliveryTasks" +CAPABILITY_CAMPAIGNS_RETENTION = "campaigns.retention" + + +@dataclass(frozen=True, slots=True) +class CampaignMailPolicyContext: + id: str + tenant_id: str + owner_user_id: str | None = None + owner_group_id: str | None = None + mail_profile_policy: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class CampaignPolicyContext: + id: str + tenant_id: str + owner_user_id: str | None = None + owner_group_id: str | None = None + settings: Mapping[str, object] = field(default_factory=dict) + + +@runtime_checkable +class CampaignMailPolicyContextProvider(Protocol): + def get_campaign_mail_policy_context( + self, + session: object, + *, + tenant_id: str, + campaign_id: str, + ) -> CampaignMailPolicyContext | None: + ... + + def set_campaign_mail_profile_policy( + self, + session: object, + *, + tenant_id: str, + campaign_id: str, + policy: Mapping[str, object], + ) -> Mapping[str, object]: + ... + + +@runtime_checkable +class CampaignAccessProvider(Protocol): + def campaign_exists(self, session: object, *, tenant_id: str, campaign_id: str) -> bool: + ... + + def can_read_campaign( + self, + session: object, + *, + tenant_id: str, + campaign_id: str, + user_id: str, + group_ids: Iterable[str] = (), + tenant_admin: bool = False, + ) -> bool: + ... + + +@runtime_checkable +class CampaignPolicyContextProvider(Protocol): + def get_campaign_policy_context( + self, + session: object, + *, + tenant_id: str | None = None, + campaign_id: str, + ) -> CampaignPolicyContext | None: + ... + + def set_campaign_settings( + self, + session: object, + *, + tenant_id: str, + campaign_id: str, + settings: Mapping[str, object], + ) -> Mapping[str, object]: + ... + + +@runtime_checkable +class CampaignDeliveryTaskProvider(Protocol): + def send_campaign_job(self, session: object, *, job_id: str, enqueue_imap_task: bool = True) -> Mapping[str, object]: + ... + + def append_sent_for_job(self, session: object, *, job_id: str) -> Mapping[str, object]: + ... + + +@runtime_checkable +class CampaignRetentionProvider(Protocol): + def apply_retention( + self, + session: object, + *, + dry_run: bool, + now: datetime, + policy_for_campaign_id: Callable[[str | None], object], + ) -> Mapping[str, Mapping[str, int]]: + ... diff --git a/src/govoplan_core/core/discovery.py b/src/govoplan_core/core/discovery.py index 6c9e7f2..157fe43 100644 --- a/src/govoplan_core/core/discovery.py +++ b/src/govoplan_core/core/discovery.py @@ -31,13 +31,20 @@ def discover_module_manifests( *, enabled_modules: Iterable[str] | None = None, group: str = ENTRY_POINT_GROUP, + ignore_load_errors: bool = False, ) -> tuple[ModuleManifest, ...]: enabled = set(enabled_modules) if enabled_modules is not None else None manifests: list[ModuleManifest] = [] for entry_point in iter_module_entry_points(group): if enabled is not None and entry_point.name not in enabled: continue - manifest = _load_manifest(entry_point) + try: + manifest = _load_manifest(entry_point) + except ModuleNotFoundError as exc: + module_root = entry_point.module.split(".", 1)[0] + if ignore_load_errors and exc.name == module_root: + continue + raise if enabled is not None and manifest.id not in enabled: continue manifests.append(manifest) diff --git a/src/govoplan_core/core/lifecycle.py b/src/govoplan_core/core/lifecycle.py new file mode 100644 index 0000000..f1491d3 --- /dev/null +++ b/src/govoplan_core/core/lifecycle.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from threading import RLock + +from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status + +from govoplan_core.core.module_management import ModuleManagementError, REQUIRED_PLATFORM_MODULES, plan_desired_enabled_modules +from govoplan_core.core.modules import ModuleContext, ModuleManifest +from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.core.runtime import configure_runtime + + +@dataclass(frozen=True, slots=True) +class ModuleLifecycleResult: + enabled_modules: tuple[str, ...] + activated_modules: tuple[str, ...] + deactivated_modules: tuple[str, ...] + mounted_modules: tuple[str, ...] + migrations_applied: bool = False + + +def require_module_active(module_id: str): + def dependency(request: Request) -> None: + registry = getattr(request.app.state, "govoplan_registry", None) + if isinstance(registry, PlatformRegistry) and registry.has_module(module_id): + return + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Module is disabled: {module_id}") + + return dependency + + +class ModuleLifecycleManager: + def __init__( + self, + *, + registry: PlatformRegistry, + available_modules: Mapping[str, ModuleManifest], + settings: object | None, + api_prefix: str = "/api/v1", + manifest_factories: Sequence[object] = (), + module_context_data: Mapping[str, object] | None = None, + ) -> None: + self.registry = registry + self.available_modules = dict(available_modules) + self.settings = settings + self.api_prefix = api_prefix + self.manifest_factories = tuple(manifest_factories) + self.context = ModuleContext(registry=registry, settings=settings, data=dict(module_context_data or {})) + self._app: FastAPI | None = None + self._mounted_modules: set[str] = set() + self._lock = RLock() + + def attach_app(self, app: FastAPI) -> None: + self._app = app + app.state.govoplan_lifecycle = self + + def configure_runtime(self) -> None: + configure_runtime(self.context) + self.registry.configure_capability_context(self.context) + + def active_module_ids(self) -> tuple[str, ...]: + return tuple(manifest.id for manifest in self.registry.manifests()) + + def mounted_module_ids(self) -> tuple[str, ...]: + return tuple(sorted(self._mounted_modules)) + + def apply_enabled_modules( + self, + requested_enabled: Sequence[str], + *, + migrate: bool = True, + protected_modules: Sequence[str] = REQUIRED_PLATFORM_MODULES, + ) -> ModuleLifecycleResult: + with self._lock: + plan = plan_desired_enabled_modules( + requested_enabled, + self.available_modules, + protected_modules=protected_modules, + ) + previous = self.active_module_ids() + previous_set = set(previous) + next_set = set(plan.enabled_modules) + activated = tuple(module_id for module_id in plan.enabled_modules if module_id not in previous_set) + deactivated = tuple(module_id for module_id in previous if module_id not in next_set) + + if migrate: + self._migrate(plan.enabled_modules) + + mounted = tuple(module_id for module_id in plan.enabled_modules if self._mount_module_router(module_id)) + + old_manifests = {manifest.id: manifest for manifest in self.registry.manifests()} + for module_id in deactivated: + hook = old_manifests[module_id].on_deactivate + if hook is not None: + hook(self.context) + + self.registry.replace(self.available_modules[module_id] for module_id in plan.enabled_modules) + self.configure_runtime() + + for module_id in activated: + hook = self.available_modules[module_id].on_activate + if hook is not None: + hook(self.context) + + if self._app is not None: + self._app.openapi_schema = None + + return ModuleLifecycleResult( + enabled_modules=plan.enabled_modules, + activated_modules=activated, + deactivated_modules=deactivated, + mounted_modules=mounted, + migrations_applied=migrate, + ) + + def _mount_module_router(self, module_id: str) -> bool: + if module_id in self._mounted_modules: + return False + app = self._app + if app is None: + raise ModuleManagementError("Module lifecycle manager is not attached to the FastAPI app.") + manifest = self.available_modules.get(module_id) + if manifest is None or manifest.route_factory is None: + self._mounted_modules.add(module_id) + return False + + module_router = manifest.route_factory(self.context) + guarded = APIRouter(dependencies=[Depends(require_module_active(module_id))]) + guarded.include_router(module_router) + app.include_router(guarded, prefix=self.api_prefix) + app.openapi_schema = None + self._mounted_modules.add(module_id) + return True + + def _migrate(self, enabled_modules: Sequence[str]) -> None: + from govoplan_core.db.migrations import migrate_database + + database_url = str(getattr(self.settings, "database_url", "") or "") + migrate_database( + database_url=database_url or None, + enabled_modules=enabled_modules, + manifest_factories=self.manifest_factories, + ) diff --git a/src/govoplan_core/core/maintenance.py b/src/govoplan_core/core/maintenance.py new file mode 100644 index 0000000..5fbb87e --- /dev/null +++ b/src/govoplan_core/core/maintenance.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +from sqlalchemy.orm import Session + +from govoplan_core.admin.models import SystemSettings +from govoplan_core.admin.settings import SYSTEM_SETTINGS_ID, get_system_settings + +MAINTENANCE_SETTINGS_KEY = "maintenance_mode" +MAINTENANCE_ACCESS_SCOPE = "system:maintenance:access" +DEFAULT_MAINTENANCE_MESSAGE = "GovOPlaN is currently in maintenance mode." + + +@dataclass(frozen=True, slots=True) +class MaintenanceMode: + enabled: bool = False + message: str | None = None + + def as_dict(self) -> dict[str, object]: + return { + "enabled": self.enabled, + "message": self.message, + } + + +def maintenance_mode_from_settings(settings: Mapping[str, object]) -> MaintenanceMode: + raw = settings.get(MAINTENANCE_SETTINGS_KEY) + if not isinstance(raw, Mapping): + return MaintenanceMode() + return MaintenanceMode( + enabled=bool(raw.get("enabled")), + message=_clean_message(raw.get("message")), + ) + + +def saved_maintenance_mode(session: Session) -> MaintenanceMode: + item = session.get(SystemSettings, SYSTEM_SETTINGS_ID) + if item is None: + return MaintenanceMode() + return maintenance_mode_from_settings(item.settings or {}) + + +def save_maintenance_mode(session: Session, mode: MaintenanceMode) -> MaintenanceMode: + item = get_system_settings(session) + settings = dict(item.settings or {}) + settings[MAINTENANCE_SETTINGS_KEY] = mode.as_dict() + item.settings = settings + session.add(item) + session.flush() + return mode + + +def maintenance_response_detail(mode: MaintenanceMode) -> dict[str, object]: + return { + "code": "maintenance_mode", + "message": mode.message or DEFAULT_MAINTENANCE_MESSAGE, + "required_scope": MAINTENANCE_ACCESS_SCOPE, + } + + +def _clean_message(value: object) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None diff --git a/src/govoplan_core/core/migrations.py b/src/govoplan_core/core/migrations.py index c5dec7e..2e5c7ee 100644 --- a/src/govoplan_core/core/migrations.py +++ b/src/govoplan_core/core/migrations.py @@ -15,15 +15,17 @@ class MigrationMetadataPlan: def migration_metadata_plan(registry: PlatformRegistry, *, extra_metadata: Iterable[MetaData] = ()) -> MigrationMetadataPlan: - metadata = list(extra_metadata) + metadata: list[MetaData] = [] script_locations: list[str] = [] + for item in extra_metadata: + if item not in metadata: + metadata.append(item) for manifest in registry.manifests(): spec = manifest.migration_spec if spec is None: continue - if isinstance(spec.metadata, MetaData): + if isinstance(spec.metadata, MetaData) and spec.metadata not in metadata: metadata.append(spec.metadata) if spec.script_location: script_locations.append(spec.script_location) return MigrationMetadataPlan(metadata=tuple(metadata), script_locations=tuple(script_locations)) - diff --git a/src/govoplan_core/core/module_guards.py b/src/govoplan_core/core/module_guards.py new file mode 100644 index 0000000..1574cf3 --- /dev/null +++ b/src/govoplan_core/core/module_guards.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from collections.abc import Callable + +from sqlalchemy import func, inspect, select + +from govoplan_core.core.modules import MigrationRetirementPlan, ModuleUninstallGuardResult + + +def persistent_table_uninstall_guard( + *models: object, + label: str | None = None, +) -> Callable[[object | None, str], tuple[ModuleUninstallGuardResult, ...]]: + def guard(session: object | None, module_id: str) -> tuple[ModuleUninstallGuardResult, ...]: + if session is None or not hasattr(session, "execute") or not hasattr(session, "get_bind"): + return ( + ModuleUninstallGuardResult( + "warning", + "uninstall_guard_no_session", + "Persistent-data uninstall guard could not inspect the database without a session.", + ), + ) + bind = session.get_bind() # type: ignore[attr-defined] + inspector = inspect(bind) + populated_tables: list[str] = [] + for model in models: + table = getattr(model, "__table__", None) + if table is None or not inspector.has_table(table.name): + continue + row_count = int(session.execute(select(func.count()).select_from(table)).scalar_one()) # type: ignore[attr-defined] + if row_count: + populated_tables.append(f"{table.name} ({row_count})") + if not populated_tables: + return () + table_summary = ", ".join(populated_tables[:8]) + if len(populated_tables) > 8: + table_summary += f", and {len(populated_tables) - 8} more" + return ( + ModuleUninstallGuardResult( + "warning", + "persistent_data_present", + f"{label or module_id} owns persistent data: {table_summary}. Non-destructive uninstall leaves this data dormant until the module is reinstalled or an explicit retirement provider handles it.", + ), + ) + + return guard + + +def drop_table_retirement_provider( + *models: object, + label: str | None = None, +) -> Callable[[object | None, str], MigrationRetirementPlan]: + def provider(session: object | None, module_id: str) -> MigrationRetirementPlan: + module_label = label or module_id + tables = tuple(_model_tables(models)) + table_names = tuple(table.name for table in tables) + if session is None or not hasattr(session, "get_bind"): + return MigrationRetirementPlan( + supported=False, + summary=f"{module_label} table retirement could not inspect the database without a session.", + blocking_reason="No database session is available.", + ) + if not tables: + return MigrationRetirementPlan( + supported=True, + summary=f"{module_label} declares no module-owned tables to retire.", + destroy_data_supported=True, + destroy_data_summary=f"{module_label} has no module-owned tables to drop.", + destroy_data_executor=lambda _session, _module_id: None, + ) + bind = session.get_bind() # type: ignore[attr-defined] + inspector = inspect(bind) + existing = tuple(table for table in tables if inspector.has_table(table.name)) + existing_names = tuple(table.name for table in existing) + missing_names = tuple(name for name in table_names if name not in existing_names) + summary = f"{module_label} can retire {len(existing_names)} table(s): {', '.join(existing_names) if existing_names else 'none currently present'}." + warnings: list[str] = [] + if missing_names: + warnings.append("Tables not present and therefore skipped: " + ", ".join(missing_names)) + + def executor(execute_session: object, _module_id: str) -> None: + if not hasattr(execute_session, "get_bind"): + raise RuntimeError("No database session is available for destructive table retirement.") + execute_bind = execute_session.get_bind() # type: ignore[attr-defined] + live_inspector = inspect(execute_bind) + live_tables = [table for table in tables if live_inspector.has_table(table.name)] + if not live_tables: + return + metadata = live_tables[0].metadata + metadata.drop_all(bind=execute_bind, tables=live_tables, checkfirst=True) + + return MigrationRetirementPlan( + supported=True, + summary=summary, + warnings=tuple(warnings), + destroy_data_supported=True, + destroy_data_summary=( + f"Destroying data will drop {module_label} table(s): " + + (", ".join(existing_names) if existing_names else "none currently present") + + "." + ), + destroy_data_warnings=( + "This permanently deletes module-owned rows unless the installer rollback restores the database snapshot.", + ), + destroy_data_executor=executor, + ) + + return provider + + +def _model_tables(models: tuple[object, ...]) -> tuple[object, ...]: + tables: list[object] = [] + for model in models: + table = getattr(model, "__table__", None) + if table is not None: + tables.append(table) + return tuple(tables) diff --git a/src/govoplan_core/core/module_installer.py b/src/govoplan_core/core/module_installer.py new file mode 100644 index 0000000..b612a04 --- /dev/null +++ b/src/govoplan_core/core/module_installer.py @@ -0,0 +1,1667 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from contextlib import AbstractContextManager +from dataclasses import dataclass +from datetime import UTC, datetime +from importlib import metadata +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import subprocess +import sys +import tomllib +from typing import Any, Literal +import time +import urllib.error +import urllib.request + +from sqlalchemy.orm import Session + +from govoplan_core.core.maintenance import saved_maintenance_mode +from govoplan_core.core.module_management import ( + PROTECTED_MODULES, + ModuleInstallPlan, + ModuleInstallPlanItem, + desired_modules_after_package_plan, + module_dependents, + module_install_plan_commands, + save_desired_enabled_modules, + save_module_install_plan, +) +from govoplan_core.core.modules import ModuleManifest, MigrationRetirementPlan + +IssueSeverity = Literal["blocker", "warning", "info"] +ChecklistStatus = Literal["done", "pending", "blocked", "warning"] +SUPPORTED_MANIFEST_CONTRACT_VERSION = "1" + + +@dataclass(frozen=True, slots=True) +class ModuleInstallerIssue: + severity: IssueSeverity + code: str + message: str + module_id: str | None = None + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "severity": self.severity, + "code": self.code, + "message": self.message, + } + if self.module_id: + payload["module_id"] = self.module_id + return payload + + +@dataclass(frozen=True, slots=True) +class ModuleInstallChecklistItem: + id: str + label: str + status: ChecklistStatus + detail: str | None = None + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "id": self.id, + "label": self.label, + "status": self.status, + } + if self.detail: + payload["detail"] = self.detail + return payload + + +@dataclass(frozen=True, slots=True) +class ModuleInstallerPreflight: + allowed: bool + maintenance_mode: bool + frontend_rebuild_required: bool + restart_required: bool + commands: tuple[str, ...] + rollback_commands: tuple[str, ...] + issues: tuple[ModuleInstallerIssue, ...] + checklist: tuple[ModuleInstallChecklistItem, ...] = () + + def as_dict(self) -> dict[str, object]: + return { + "allowed": self.allowed, + "maintenance_mode": self.maintenance_mode, + "frontend_rebuild_required": self.frontend_rebuild_required, + "restart_required": self.restart_required, + "commands": list(self.commands), + "rollback_commands": list(self.rollback_commands), + "issues": [issue.as_dict() for issue in self.issues], + "checklist": [item.as_dict() for item in self.checklist], + } + + +@dataclass(frozen=True, slots=True) +class ModuleInstallerRunResult: + run_id: str + status: Literal["dry-run", "applied", "failed", "rolled-back"] + record_path: Path + commands: tuple[str, ...] + return_code: int = 0 + error: str | None = None + rollback: dict[str, object] | None = None + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "run_id": self.run_id, + "status": self.status, + "record_path": str(self.record_path), + "commands": list(self.commands), + "return_code": self.return_code, + } + if self.error: + payload["error"] = self.error + if self.rollback: + payload["rollback"] = self.rollback + return payload + + +def default_installer_runtime_dir(database_url: str | None = None, *, cwd: Path | None = None) -> Path: + root = cwd or Path.cwd() + if database_url and database_url.startswith("sqlite:///"): + raw_path = database_url[len("sqlite:///"):] + if raw_path and raw_path != ":memory:": + db_path = Path(raw_path).expanduser() + if not db_path.is_absolute(): + db_path = root / db_path + return db_path.parent / "module-installer" + return root / "runtime" / "module-installer" + + +def module_install_preflight( + *, + plan: ModuleInstallPlan, + available: Mapping[str, ModuleManifest], + current_enabled: Iterable[str], + desired_enabled: Iterable[str], + maintenance_mode: bool, + session: object | None = None, + webui_root: Path | None = None, + runtime_dir: Path | None = None, +) -> ModuleInstallerPreflight: + issues: list[ModuleInstallerIssue] = [] + current = {item for item in current_enabled} + desired = {item for item in desired_enabled} + planned_items = tuple(item for item in plan.items if item.status == "planned") + dependents = module_dependents(available) + + if not planned_items: + issues.append(ModuleInstallerIssue("warning", "empty_plan", "No planned package changes are present.")) + if not maintenance_mode: + issues.append(ModuleInstallerIssue("blocker", "maintenance_required", "Package changes require maintenance mode.")) + + issues.extend(module_manifest_compatibility_issues(available)) + + frontend_rebuild_required = False + for item in planned_items: + if item.action == "uninstall": + issues.extend(_uninstall_guard_issues( + item, + current=current, + desired=desired, + dependents=dependents, + available=available, + session=session, + )) + if item.action == "install": + if item.module_id not in available: + issues.append(ModuleInstallerIssue( + "info", + "compatibility_checked_after_install", + "Module manifest compatibility can be fully checked after the package is installed and discovered.", + item.module_id, + )) + if item.python_ref and not item.python_package: + issues.append(ModuleInstallerIssue( + "blocker", + "python_package_required", + "Python installs must include the distribution package name so rollback can uninstall newly added packages.", + item.module_id, + )) + if item.python_ref and not _looks_pinned_dependency_ref(item.python_ref): + issues.append(ModuleInstallerIssue( + "blocker", + "unpinned_python_ref", + "Python install refs must be pinned to an exact version or tagged git ref.", + item.module_id, + )) + if item.webui_ref and not _looks_pinned_dependency_ref(item.webui_ref): + issues.append(ModuleInstallerIssue( + "blocker", + "unpinned_webui_ref", + "WebUI install refs must be pinned to an exact version or tagged git ref.", + item.module_id, + )) + if item.webui_package or item.webui_ref: + frontend_rebuild_required = True + + if frontend_rebuild_required: + if webui_root is None: + issues.append(ModuleInstallerIssue("blocker", "webui_root_missing", "WebUI package changes need a webui root for npm install/build.")) + elif not (webui_root / "package.json").exists(): + issues.append(ModuleInstallerIssue("blocker", "webui_package_missing", f"WebUI package.json does not exist at {webui_root}.")) + + record_dir = runtime_dir or default_installer_runtime_dir() + rollback_commands = ( + f"govoplan-module-installer --runtime-dir {record_dir} --rollback ", + ) + blocker = any(issue.severity == "blocker" for issue in issues) + checklist = _post_install_checklist( + planned_items=planned_items, + maintenance_mode=maintenance_mode, + frontend_rebuild_required=frontend_rebuild_required, + blocker=blocker, + ) + return ModuleInstallerPreflight( + allowed=not blocker and bool(planned_items), + maintenance_mode=maintenance_mode, + frontend_rebuild_required=frontend_rebuild_required, + restart_required=bool(planned_items), + commands=module_install_plan_commands(planned_items), + rollback_commands=rollback_commands, + issues=tuple(issues), + checklist=checklist, + ) + + +def run_module_install_plan( + *, + session: Session, + plan: ModuleInstallPlan, + available: Mapping[str, ModuleManifest], + current_enabled: Iterable[str], + desired_enabled: Iterable[str], + database_url: str, + runtime_dir: Path | None = None, + webui_root: Path | None = None, + npm_bin: str = "npm", + build_webui: bool = False, + migrate_database: bool = False, + database_backup_command: str | None = None, + database_restore_command: str | None = None, + database_restore_check_command: str | None = None, + activate_installed_modules: bool = True, + remove_uninstalled_modules_from_desired: bool = True, + dry_run: bool = False, +) -> ModuleInstallerRunResult: + maintenance_mode = saved_maintenance_mode(session) + effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url) + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=current_enabled, + desired_enabled=desired_enabled, + maintenance_mode=maintenance_mode.enabled, + session=session, + webui_root=webui_root, + runtime_dir=effective_runtime_dir, + ) + if not preflight.allowed: + raise ModuleInstallerError("Install preflight is blocked: " + "; ".join(issue.message for issue in preflight.issues if issue.severity == "blocker")) + + run_id = _run_id() + run_dir = effective_runtime_dir / "runs" / run_id + run_dir.mkdir(parents=True, exist_ok=False) + record_path = run_dir / "record.json" + commands = structured_install_commands( + plan, + webui_root=webui_root, + npm_bin=npm_bin, + build_webui=build_webui, + migrate_database_url=database_url if migrate_database else None, + verify_modules=True, + ) + snapshot = _snapshot_environment( + run_dir, + webui_root=webui_root, + database_url=database_url, + include_database=migrate_database or _destructive_retirement_requested(plan), + database_backup_command=database_backup_command, + database_restore_command=database_restore_command, + database_restore_check_command=database_restore_check_command, + ) + + record: dict[str, Any] = { + "run_id": run_id, + "started_at": datetime.now(tz=UTC).isoformat(), + "status": "dry-run" if dry_run else "running", + "plan": [item.as_dict() for item in plan.items], + "preflight": preflight.as_dict(), + "snapshot": snapshot, + "commands": [_command_record(command) for command in commands], + "build_webui": build_webui, + "migrate_database": migrate_database, + "activate_installed_modules": activate_installed_modules, + "remove_uninstalled_modules_from_desired": remove_uninstalled_modules_from_desired, + } + _write_json(record_path, record) + + if dry_run: + return ModuleInstallerRunResult(run_id=run_id, status="dry-run", record_path=record_path, commands=tuple(command["display"] for command in commands)) + + executed: list[dict[str, object]] = [] + failed_error: str | None = None + with _installer_lock(effective_runtime_dir): + try: + retirements = _execute_destructive_retirements(session=session, plan=plan, available=available) + if retirements: + session.commit() + record["retirements"] = retirements + _write_json(record_path, record) + for command in commands: + result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False) + executed.append({ + **_command_record(command), + "return_code": result.returncode, + "stdout": result.stdout[-8000:], + "stderr": result.stderr[-8000:], + }) + record["commands"] = executed + _write_json(record_path, record) + if result.returncode != 0: + raise ModuleInstallerError(f"Command failed ({result.returncode}): {command['display']}") + except Exception as exc: + failed_error = str(exc) + try: + session.rollback() + except Exception: + pass + + if failed_error is not None: + record.update({ + "status": "failed", + "finished_at": datetime.now(tz=UTC).isoformat(), + "error": failed_error, + "commands": executed, + }) + _write_json(record_path, record) + if _destructive_retirement_requested(plan): + rollback = rollback_module_install_run( + run_id=run_id, + runtime_dir=effective_runtime_dir, + npm_bin=npm_bin, + build_webui=build_webui, + database_restore_command=database_restore_command, + database_url=database_url, + ) + _update_run_record(record_path, {"destructive_retirement_rollback": rollback.as_dict()}) + return ModuleInstallerRunResult( + run_id=run_id, + status="rolled-back" if rollback.return_code == 0 else "failed", + record_path=record_path, + commands=tuple(command["display"] for command in commands), + return_code=1, + error=failed_error, + rollback=rollback.as_dict(), + ) + return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1, error=failed_error) + + applied_items = tuple(_mark_applied(item) for item in plan.items) + save_module_install_plan(session, applied_items) + if activate_installed_modules or remove_uninstalled_modules_from_desired: + next_desired = desired_modules_after_package_plan( + desired_enabled, + plan, + activate_installed_modules=activate_installed_modules, + remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, + ) + save_desired_enabled_modules(session, next_desired) + record["desired_enabled_after"] = list(next_desired) + session.commit() + record.update({ + "status": "applied", + "finished_at": datetime.now(tz=UTC).isoformat(), + "commands": executed, + }) + _write_json(record_path, record) + return ModuleInstallerRunResult(run_id=run_id, status="applied", record_path=record_path, commands=tuple(command["display"] for command in commands)) + + +def supervise_module_install_plan( + *, + session: Session, + plan: ModuleInstallPlan, + available: Mapping[str, ModuleManifest], + current_enabled: Iterable[str], + desired_enabled: Iterable[str], + database_url: str, + runtime_dir: Path | None = None, + webui_root: Path | None = None, + npm_bin: str = "npm", + build_webui: bool = False, + migrate_database: bool = False, + database_backup_command: str | None = None, + database_restore_command: str | None = None, + database_restore_check_command: str | None = None, + activate_installed_modules: bool = True, + remove_uninstalled_modules_from_desired: bool = True, + restart_command: str | None = None, + restart_commands: Iterable[str] | None = None, + health_url: str | None = None, + health_urls: Iterable[str] | None = None, + health_timeout_seconds: float = 60.0, + health_interval_seconds: float = 2.0, +) -> ModuleInstallerRunResult: + effective_runtime_dir = runtime_dir or default_installer_runtime_dir(database_url) + effective_restart_commands = _command_list(restart_command, restart_commands) + effective_health_urls = _command_list(health_url, health_urls) + result = run_module_install_plan( + session=session, + plan=plan, + available=available, + current_enabled=current_enabled, + desired_enabled=desired_enabled, + database_url=database_url, + runtime_dir=effective_runtime_dir, + webui_root=webui_root, + npm_bin=npm_bin, + build_webui=build_webui, + migrate_database=migrate_database, + database_backup_command=database_backup_command, + database_restore_command=database_restore_command, + database_restore_check_command=database_restore_check_command, + activate_installed_modules=activate_installed_modules, + remove_uninstalled_modules_from_desired=remove_uninstalled_modules_from_desired, + dry_run=False, + ) + supervisor: dict[str, object] = { + "started_at": datetime.now(tz=UTC).isoformat(), + "restart_commands": list(effective_restart_commands), + "health_urls": list(effective_health_urls), + } + _update_run_record(result.record_path, {"supervisor": supervisor}) + + if result.return_code != 0: + if result.rollback is not None: + return result + return _rollback_after_supervisor_failure( + result=result, + session=session, + original_plan=plan, + original_desired_enabled=tuple(desired_enabled), + runtime_dir=effective_runtime_dir, + npm_bin=npm_bin, + restart_commands=effective_restart_commands, + health_urls=effective_health_urls, + health_timeout_seconds=health_timeout_seconds, + health_interval_seconds=health_interval_seconds, + build_webui=build_webui, + reason=result.error or "Installer command failed.", + ) + + restart_results = _run_restart_commands(effective_restart_commands) + if restart_results: + supervisor["restart_commands"] = restart_results + _update_run_record(result.record_path, {"supervisor": supervisor}) + if any(item.get("return_code") != 0 for item in restart_results): + return _rollback_after_supervisor_failure( + result=result, + session=session, + original_plan=plan, + original_desired_enabled=tuple(desired_enabled), + runtime_dir=effective_runtime_dir, + npm_bin=npm_bin, + restart_commands=effective_restart_commands, + health_urls=effective_health_urls, + health_timeout_seconds=health_timeout_seconds, + health_interval_seconds=health_interval_seconds, + build_webui=build_webui, + reason="Restart command failed.", + ) + + if effective_health_urls: + health = _wait_for_health_urls( + effective_health_urls, + timeout_seconds=health_timeout_seconds, + interval_seconds=health_interval_seconds, + ) + supervisor["health"] = health + _update_run_record(result.record_path, {"supervisor": supervisor}) + if not health.get("ok"): + return _rollback_after_supervisor_failure( + result=result, + session=session, + original_plan=plan, + original_desired_enabled=tuple(desired_enabled), + runtime_dir=effective_runtime_dir, + npm_bin=npm_bin, + restart_commands=effective_restart_commands, + health_urls=effective_health_urls, + health_timeout_seconds=health_timeout_seconds, + health_interval_seconds=health_interval_seconds, + build_webui=build_webui, + reason=str(health.get("error") or "Health check failed."), + ) + + supervisor.update({ + "status": "ok", + "finished_at": datetime.now(tz=UTC).isoformat(), + }) + _update_run_record(result.record_path, {"supervisor": supervisor}) + return result + + +def rollback_module_install_run( + *, + run_id: str, + runtime_dir: Path, + npm_bin: str = "npm", + build_webui: bool = False, + database_restore_command: str | None = None, + database_url: str | None = None, +) -> ModuleInstallerRunResult: + run_dir = runtime_dir / "runs" / run_id + record_path = run_dir / "record.json" + if not record_path.exists(): + raise ModuleInstallerError(f"Run record does not exist: {record_path}") + record = json.loads(record_path.read_text(encoding="utf-8")) + snapshot = record.get("snapshot") if isinstance(record.get("snapshot"), dict) else {} + commands: list[dict[str, Any]] = [] + for package in _planned_python_install_packages(record): + commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", package])) + pip_freeze = snapshot.get("pip_freeze") + if isinstance(pip_freeze, str) and (run_dir / pip_freeze).exists(): + commands.append(_structured_command([sys.executable, "-m", "pip", "install", "-r", str(run_dir / pip_freeze)])) + webui_root = snapshot.get("webui_root") + if isinstance(webui_root, str): + root = Path(webui_root) + _restore_snapshot_file(run_dir, snapshot, "package_json", root / "package.json") + _restore_snapshot_file(run_dir, snapshot, "package_lock", root / "package-lock.json") + if (root / "package.json").exists(): + commands.append(_structured_command([npm_bin, "install"], cwd=root)) + if build_webui or bool(record.get("build_webui")): + commands.append(_structured_command([npm_bin, "run", "build"], cwd=root)) + + with _installer_lock(runtime_dir): + executed: list[dict[str, object]] = [] + for command in commands: + result = subprocess.run(command["argv"], cwd=command.get("cwd"), text=True, capture_output=True, check=False) + executed.append({ + **_command_record(command), + "return_code": result.returncode, + "stdout": result.stdout[-8000:], + "stderr": result.stderr[-8000:], + }) + if result.returncode != 0: + record.update({"rollback_status": "failed", "rollback_commands": executed}) + _write_json(record_path, record) + return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1) + database_restore = _restore_database_snapshot( + run_dir, + snapshot, + database_restore_command=database_restore_command, + database_url=database_url, + ) + if database_restore is not None and not database_restore.get("restored"): + record.update({ + "rollback_status": "failed", + "rollback_commands": executed, + "rollback_database_restore": database_restore, + }) + _write_json(record_path, record) + return ModuleInstallerRunResult(run_id=run_id, status="failed", record_path=record_path, commands=tuple(command["display"] for command in commands), return_code=1, error=str(database_restore.get("error") or "Database restore failed.")) + + record.update({ + "rollback_status": "rolled-back", + "rolled_back_at": datetime.now(tz=UTC).isoformat(), + "rollback_commands": executed, + }) + if database_restore is not None: + record["rollback_database_restore"] = database_restore + _write_json(record_path, record) + return ModuleInstallerRunResult(run_id=run_id, status="rolled-back", record_path=record_path, commands=tuple(command["display"] for command in commands)) + + +def list_module_installer_runs(*, runtime_dir: Path, limit: int = 25) -> tuple[dict[str, object], ...]: + runs_dir = runtime_dir / "runs" + if not runs_dir.exists(): + return () + records: list[dict[str, object]] = [] + for record_path in sorted(runs_dir.glob("*/record.json"), reverse=True): + record = _read_run_record(record_path) + if record is None: + continue + records.append(_run_summary(record_path, record)) + if len(records) >= limit: + break + return tuple(records) + + +def read_module_installer_run(*, runtime_dir: Path, run_id: str) -> dict[str, object]: + if not re.match(r"^[A-Za-z0-9_.:-]+$", run_id): + raise ModuleInstallerError(f"Invalid installer run id: {run_id!r}") + record_path = runtime_dir / "runs" / run_id / "record.json" + record = _read_run_record(record_path) + if record is None: + raise ModuleInstallerError(f"Installer run record does not exist: {run_id}") + record["record_path"] = str(record_path) + return record + + +def module_installer_lock_status(*, runtime_dir: Path) -> dict[str, object]: + lock_path = runtime_dir / "install.lock" + if not lock_path.exists(): + return {"locked": False, "path": str(lock_path)} + try: + payload = json.loads(lock_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + payload = {} + except (OSError, json.JSONDecodeError): + payload = {} + return {"locked": True, "path": str(lock_path), **payload} + + +def module_installer_daemon_status(*, runtime_dir: Path) -> dict[str, object]: + status_path = _daemon_status_path(runtime_dir) + record = _read_run_record(status_path) + if record is None: + return {"running": False, "status": "unknown", "path": str(status_path)} + pid = record.get("pid") + running = isinstance(pid, int) and _process_is_running(pid) + return {**record, "running": running, "path": str(status_path)} + + +def update_module_installer_daemon_status(*, runtime_dir: Path, patch: Mapping[str, object]) -> dict[str, object]: + status_path = _daemon_status_path(runtime_dir) + record = _read_run_record(status_path) or {} + record.update({ + "pid": os.getpid(), + "updated_at": datetime.now(tz=UTC).isoformat(), + **patch, + }) + _write_json(status_path, record) + return module_installer_daemon_status(runtime_dir=runtime_dir) + + +def queue_module_installer_request( + *, + runtime_dir: Path, + options: Mapping[str, object] | None = None, + requested_by: str | None = None, + retry_of: str | None = None, +) -> dict[str, object]: + request_id = _run_id() + record = { + "request_id": request_id, + "status": "queued", + "created_at": datetime.now(tz=UTC).isoformat(), + "requested_by": requested_by, + "options": dict(options or {}), + } + if retry_of: + record["retry_of"] = retry_of + with _request_queue_lock(runtime_dir): + path = _request_record_path(runtime_dir, request_id) + _write_json(path, record) + return {**record, "record_path": str(path)} + + +def list_module_installer_requests(*, runtime_dir: Path, limit: int = 25) -> tuple[dict[str, object], ...]: + requests_dir = runtime_dir / "requests" + if not requests_dir.exists(): + return () + records: list[dict[str, object]] = [] + for record_path in sorted(requests_dir.glob("*.json"), reverse=True): + record = _read_run_record(record_path) + if record is None: + continue + record["record_path"] = str(record_path) + records.append(record) + if len(records) >= limit: + break + return tuple(records) + + +def read_module_installer_request(*, runtime_dir: Path, request_id: str) -> dict[str, object]: + if not re.match(r"^[A-Za-z0-9_.:-]+$", request_id): + raise ModuleInstallerError(f"Invalid installer request id: {request_id!r}") + record_path = _request_record_path(runtime_dir, request_id) + record = _read_run_record(record_path) + if record is None: + raise ModuleInstallerError(f"Installer request record does not exist: {request_id}") + record["record_path"] = str(record_path) + return record + + +def cancel_module_installer_request( + *, + runtime_dir: Path, + request_id: str, + cancelled_by: str | None = None, +) -> dict[str, object]: + with _request_queue_lock(runtime_dir): + record = read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id) + if record.get("status") != "queued": + raise ModuleInstallerError("Only queued installer requests can be cancelled.") + record.update({ + "status": "cancelled", + "cancelled_at": datetime.now(tz=UTC).isoformat(), + "cancelled_by": cancelled_by, + }) + path = _request_record_path(runtime_dir, request_id) + record.pop("record_path", None) + _write_json(path, record) + return read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id) + + +def retry_module_installer_request( + *, + runtime_dir: Path, + request_id: str, + requested_by: str | None = None, +) -> dict[str, object]: + record = read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id) + status = record.get("status") + if status not in {"failed", "cancelled"}: + raise ModuleInstallerError("Only failed or cancelled installer requests can be retried.") + options = record.get("options") if isinstance(record.get("options"), Mapping) else {} + return queue_module_installer_request( + runtime_dir=runtime_dir, + requested_by=requested_by, + options=options, + retry_of=request_id, + ) + + +def claim_next_module_installer_request(*, runtime_dir: Path) -> dict[str, object] | None: + with _request_queue_lock(runtime_dir): + for record_path in sorted((runtime_dir / "requests").glob("*.json")): + record = _read_run_record(record_path) + if record is None or record.get("status") != "queued": + continue + record.update({ + "status": "running", + "started_at": datetime.now(tz=UTC).isoformat(), + "worker_pid": os.getpid(), + }) + _write_json(record_path, record) + return {**record, "record_path": str(record_path)} + return None + + +def update_module_installer_request(*, runtime_dir: Path, request_id: str, patch: Mapping[str, object]) -> dict[str, object]: + with _request_queue_lock(runtime_dir): + record = read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id) + record.update(patch) + path = _request_record_path(runtime_dir, request_id) + record.pop("record_path", None) + _write_json(path, record) + return read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id) + + +def structured_install_commands( + plan: ModuleInstallPlan, + *, + webui_root: Path | None, + npm_bin: str = "npm", + build_webui: bool = False, + migrate_database_url: str | None = None, + verify_modules: bool = False, +) -> tuple[dict[str, Any], ...]: + commands: list[dict[str, Any]] = [] + webui_changed = False + for item in plan.items: + if item.status != "planned": + continue + if item.action == "install": + if item.python_ref: + commands.append(_structured_command([sys.executable, "-m", "pip", "install", item.python_ref])) + if item.webui_package and item.webui_ref and webui_root is not None: + commands.append(_structured_command([npm_bin, "pkg", "set", f"dependencies.{item.webui_package}={item.webui_ref}"], cwd=webui_root)) + webui_changed = True + elif item.action == "uninstall": + if item.python_package: + commands.append(_structured_command([sys.executable, "-m", "pip", "uninstall", "-y", item.python_package])) + if item.webui_package and webui_root is not None: + commands.append(_structured_command([npm_bin, "pkg", "delete", f"dependencies.{item.webui_package}"], cwd=webui_root)) + webui_changed = True + if webui_changed and webui_root is not None: + commands.append(_structured_command([npm_bin, "install"], cwd=webui_root)) + if build_webui: + commands.append(_structured_command([npm_bin, "run", "build"], cwd=webui_root)) + verify_command = _module_verify_command(plan) if verify_modules else None + if verify_command is not None: + commands.append(verify_command) + if migrate_database_url is not None: + commands.append(_structured_command([sys.executable, "-m", "govoplan_core.commands.init_db", "--database-url", migrate_database_url])) + return tuple(commands) + + +def _destructive_retirement_requested(plan: ModuleInstallPlan) -> bool: + return any(item.status == "planned" and item.action == "uninstall" and item.destroy_data for item in plan.items) + + +def _execute_destructive_retirements( + *, + session: Session, + plan: ModuleInstallPlan, + available: Mapping[str, ModuleManifest], +) -> list[dict[str, object]]: + retirements: list[dict[str, object]] = [] + for item in plan.items: + if item.status != "planned" or item.action != "uninstall" or not item.destroy_data: + continue + manifest = available.get(item.module_id) + migration = manifest.migration_spec if manifest is not None else None + provider = migration.retirement_provider if migration is not None else None + if provider is None: + raise ModuleInstallerError(f"Module {item.module_id!r} does not provide a destructive retirement provider.") + retirement_plan = provider(session, item.module_id) + if not retirement_plan.supported: + raise ModuleInstallerError(retirement_plan.blocking_reason or retirement_plan.summary or f"Module {item.module_id!r} cannot be retired.") + if not retirement_plan.destroy_data_supported or retirement_plan.destroy_data_executor is None: + raise ModuleInstallerError(f"Module {item.module_id!r} does not support destructive data retirement.") + retirement_plan.destroy_data_executor(session, item.module_id) + retirements.append({ + "module_id": item.module_id, + "summary": retirement_plan.destroy_data_summary or retirement_plan.summary, + "warnings": list(retirement_plan.destroy_data_warnings), + }) + return retirements + + +class ModuleInstallerError(RuntimeError): + pass + + +def _uninstall_guard_issues( + item: ModuleInstallPlanItem, + *, + current: set[str], + desired: set[str], + dependents: Mapping[str, tuple[str, ...]], + available: Mapping[str, ModuleManifest], + session: object | None, +) -> tuple[ModuleInstallerIssue, ...]: + issues: list[ModuleInstallerIssue] = [] + if item.module_id in PROTECTED_MODULES: + issues.append(ModuleInstallerIssue("blocker", "protected_module", "Protected platform modules cannot be uninstalled.", item.module_id)) + if item.module_id in current: + issues.append(ModuleInstallerIssue("blocker", "module_active", "Disable the module before uninstalling its package.", item.module_id)) + if item.module_id in desired: + issues.append(ModuleInstallerIssue("blocker", "module_desired", "Remove the module from saved startup state before uninstalling its package.", item.module_id)) + active_dependents = sorted(module_id for module_id in dependents.get(item.module_id, ()) if module_id in current or module_id in desired) + if active_dependents: + issues.append(ModuleInstallerIssue("blocker", "active_dependents", "Disable dependent modules first: " + ", ".join(active_dependents), item.module_id)) + manifest = available.get(item.module_id) + destroy_supported = False + if manifest is not None: + migration = manifest.migration_spec + if migration is not None: + if not migration.retirement_supported: + issues.append(ModuleInstallerIssue( + "blocker" if item.destroy_data else "warning", + "migration_retirement_not_requested", + "This module owns migrations. Default package uninstall is non-destructive: schema/data remain dormant until reinstall or explicit retirement support is added.", + item.module_id, + )) + elif migration.retirement_provider is None: + detail = f" {migration.retirement_notes}" if migration.retirement_notes else "" + issues.append(ModuleInstallerIssue( + "blocker" if item.destroy_data else "warning", + "migration_retirement_manual", + "This module declares migration retirement support but no automated retirement provider is registered." + detail, + item.module_id, + )) + else: + try: + retirement_plan = migration.retirement_provider(session, item.module_id) + except Exception as exc: + issues.append(ModuleInstallerIssue("blocker", "migration_retirement_check_failed", f"Migration retirement check failed: {exc}", item.module_id)) + else: + destroy_supported = retirement_plan.destroy_data_supported and retirement_plan.destroy_data_executor is not None + issues.extend(_migration_retirement_issues(retirement_plan, item.module_id, destroy_data_requested=item.destroy_data)) + elif item.destroy_data: + issues.append(ModuleInstallerIssue("blocker", "migration_retirement_missing", "This module does not declare a data retirement provider.", item.module_id)) + for provider in manifest.uninstall_guard_providers: + try: + for result in provider(session, item.module_id): + if item.destroy_data and destroy_supported and result.code == "persistent_data_present": + continue + issues.append(ModuleInstallerIssue(result.severity, result.code, result.message, item.module_id)) + except Exception as exc: + issues.append(ModuleInstallerIssue("blocker", "uninstall_guard_failed", f"Uninstall guard failed: {exc}", item.module_id)) + return tuple(issues) + + +def _migration_retirement_issues(plan: MigrationRetirementPlan, module_id: str, *, destroy_data_requested: bool = False) -> tuple[ModuleInstallerIssue, ...]: + issues: list[ModuleInstallerIssue] = [] + if not plan.supported: + issues.append(ModuleInstallerIssue( + "blocker", + "migration_retirement_blocked", + plan.blocking_reason or plan.summary or "Migration retirement provider blocked package uninstall.", + module_id, + )) + elif plan.summary: + issues.append(ModuleInstallerIssue("info", "migration_retirement_supported", plan.summary, module_id)) + if destroy_data_requested: + if not plan.destroy_data_supported or plan.destroy_data_executor is None: + issues.append(ModuleInstallerIssue( + "blocker", + "migration_retirement_destroy_unsupported", + "Destroying module data was requested, but this module does not provide an automated destructive retirement executor.", + module_id, + )) + else: + issues.append(ModuleInstallerIssue( + "warning", + "migration_retirement_destroy_planned", + plan.destroy_data_summary or "Destructive module data retirement will run before package uninstall.", + module_id, + )) + for warning in plan.destroy_data_warnings: + issues.append(ModuleInstallerIssue("warning", "migration_retirement_destroy_warning", warning, module_id)) + for warning in plan.warnings: + issues.append(ModuleInstallerIssue("warning", "migration_retirement_warning", warning, module_id)) + return tuple(issues) + + +def _module_verify_command(plan: ModuleInstallPlan) -> dict[str, Any] | None: + installed: list[str] = [] + absent: list[str] = [] + for item in plan.items: + if item.status != "planned": + continue + if item.action == "install" and item.python_ref: + installed.append(item.module_id) + elif item.action == "uninstall" and item.python_package: + absent.append(item.module_id) + if not installed and not absent: + return None + argv = [sys.executable, "-m", "govoplan_core.commands.module_verify"] + for module_id in dict.fromkeys(installed): + argv.extend(["--installed", module_id]) + for module_id in dict.fromkeys(absent): + argv.extend(["--absent", module_id]) + return _structured_command(argv) + + +def module_manifest_compatibility_issues(available: Mapping[str, ModuleManifest]) -> tuple[ModuleInstallerIssue, ...]: + current_core_version = _current_core_version() + issues: list[ModuleInstallerIssue] = [] + for manifest in available.values(): + compatibility = manifest.compatibility + if compatibility.manifest_contract_version != SUPPORTED_MANIFEST_CONTRACT_VERSION: + issues.append(ModuleInstallerIssue( + "blocker", + "unsupported_manifest_contract", + ( + f"Module manifest contract {compatibility.manifest_contract_version} is not supported by " + f"this core runtime ({SUPPORTED_MANIFEST_CONTRACT_VERSION})." + ), + manifest.id, + )) + if compatibility.core_version_min and _compare_versions(current_core_version, compatibility.core_version_min) < 0: + issues.append(ModuleInstallerIssue( + "blocker", + "core_version_too_low", + f"Module requires core >= {compatibility.core_version_min}; current core is {current_core_version}.", + manifest.id, + )) + if compatibility.core_version_max and _compare_versions(current_core_version, compatibility.core_version_max) > 0: + issues.append(ModuleInstallerIssue( + "blocker", + "core_version_too_high", + f"Module supports core <= {compatibility.core_version_max}; current core is {current_core_version}.", + manifest.id, + )) + return tuple(issues) + + +def _current_core_version() -> str: + try: + return metadata.version("govoplan-core") + except metadata.PackageNotFoundError: + pyproject_path = Path(__file__).resolve().parents[3] / "pyproject.toml" + try: + with pyproject_path.open("rb") as handle: + project = tomllib.load(handle).get("project", {}) + version = project.get("version") + if isinstance(version, str) and version: + return version + except (OSError, tomllib.TOMLDecodeError): + pass + return "0.0.0" + + +def _compare_versions(left: str, right: str) -> int: + left_parts = _version_tuple(left) + right_parts = _version_tuple(right) + max_length = max(len(left_parts), len(right_parts)) + padded_left = left_parts + (0,) * (max_length - len(left_parts)) + padded_right = right_parts + (0,) * (max_length - len(right_parts)) + if padded_left < padded_right: + return -1 + if padded_left > padded_right: + return 1 + return 0 + + +def _version_tuple(value: str) -> tuple[int, ...]: + match = re.match(r"^\s*(\d+(?:\.\d+)*)", value) + if not match: + return (0,) + return tuple(int(part) for part in match.group(1).split(".")) + + +def _post_install_checklist( + *, + planned_items: tuple[ModuleInstallPlanItem, ...], + maintenance_mode: bool, + frontend_rebuild_required: bool, + blocker: bool, +) -> tuple[ModuleInstallChecklistItem, ...]: + has_package_changes = bool(planned_items) + has_backend_changes = any(item.python_ref or item.python_package for item in planned_items) + preflight_status: ChecklistStatus = "blocked" if blocker else "done" + package_status: ChecklistStatus = "pending" if has_package_changes else "done" + restart_status: ChecklistStatus = "pending" if has_package_changes else "done" + migration_status: ChecklistStatus = "pending" if has_backend_changes else "done" + webui_status: ChecklistStatus = "pending" if frontend_rebuild_required else "done" + module_state_status: ChecklistStatus = "pending" if has_package_changes else "done" + health_status: ChecklistStatus = "pending" if has_package_changes else "done" + return ( + ModuleInstallChecklistItem( + id="maintenance-mode", + label="Maintenance mode", + status="done" if maintenance_mode else "blocked", + detail="Enable maintenance mode before changing installed packages.", + ), + ModuleInstallChecklistItem( + id="preflight", + label="Installer preflight", + status=preflight_status, + detail="Compatibility, pinned refs, uninstall guards, and package targets are checked before apply.", + ), + ModuleInstallChecklistItem( + id="package-apply", + label="Package apply", + status=package_status, + detail="The installer snapshots Python and WebUI package state before mutating packages.", + ), + ModuleInstallChecklistItem( + id="migrations", + label="Module migrations", + status=migration_status, + detail="Run or verify module migrations after backend package changes.", + ), + ModuleInstallChecklistItem( + id="webui-rebuild", + label="WebUI rebuild", + status=webui_status, + detail="Required when WebUI package dependencies changed.", + ), + ModuleInstallChecklistItem( + id="backend-restart", + label="Backend restart/reload", + status=restart_status, + detail="Required after Python package changes so module discovery sees the new package set.", + ), + ModuleInstallChecklistItem( + id="module-state", + label="Module state activation", + status=module_state_status, + detail="Enable or disable the module after package installation/removal has succeeded.", + ), + ModuleInstallChecklistItem( + id="health-check", + label="Health check", + status=health_status, + detail="Supervisor mode polls the health endpoint and rolls back if the restarted server does not come up.", + ), + ) + + +def _rollback_after_supervisor_failure( + *, + result: ModuleInstallerRunResult, + session: Session, + original_plan: ModuleInstallPlan, + original_desired_enabled: Iterable[str], + runtime_dir: Path, + npm_bin: str, + restart_commands: Iterable[str], + health_urls: Iterable[str], + health_timeout_seconds: float, + health_interval_seconds: float, + build_webui: bool, + reason: str, +) -> ModuleInstallerRunResult: + supervisor: dict[str, object] = { + "status": "rollback-started", + "failure_reason": reason, + "rolled_back_at": datetime.now(tz=UTC).isoformat(), + } + try: + session.rollback() + except Exception: + pass + rollback = rollback_module_install_run(run_id=result.run_id, runtime_dir=runtime_dir, npm_bin=npm_bin, build_webui=build_webui) + rollback_record = _read_run_record(result.record_path) or {} + if not _database_restore_was_applied(rollback_record): + save_module_install_plan(session, original_plan.items) + save_desired_enabled_modules(session, original_desired_enabled) + session.commit() + supervisor["rollback"] = rollback.as_dict() + + rollback_restart = _run_restart_commands(restart_commands) + if rollback_restart: + supervisor["rollback_restart_commands"] = rollback_restart + if rollback_restart and all(item.get("return_code") == 0 for item in rollback_restart) and tuple(health_urls): + supervisor["rollback_health"] = _wait_for_health_urls( + health_urls, + timeout_seconds=health_timeout_seconds, + interval_seconds=health_interval_seconds, + ) + supervisor["status"] = "rolled-back" if rollback.return_code == 0 else "rollback-failed" + supervisor["finished_at"] = datetime.now(tz=UTC).isoformat() + _update_run_record(result.record_path, {"supervisor": supervisor}) + return ModuleInstallerRunResult( + run_id=result.run_id, + status="rolled-back" if rollback.return_code == 0 else "failed", + record_path=result.record_path, + commands=result.commands, + return_code=1, + error=reason, + rollback=rollback.as_dict(), + ) + + +def _command_list(first: str | None, rest: Iterable[str] | None) -> tuple[str, ...]: + commands: list[str] = [] + if first and first.strip(): + commands.append(first.strip()) + if rest: + for item in rest: + if item and str(item).strip(): + commands.append(str(item).strip()) + return tuple(dict.fromkeys(commands)) + + +def _run_restart_commands(commands: Iterable[str]) -> list[dict[str, object]]: + return [_run_restart_command(command) for command in commands] + + +def _run_restart_command(command: str) -> dict[str, object]: + if not command.strip(): + return { + "command": command, + "return_code": 0, + "stdout": "", + "stderr": "", + } + result = subprocess.run(command, shell=True, text=True, capture_output=True, check=False) + return { + "command": command, + "return_code": result.returncode, + "stdout": result.stdout[-8000:], + "stderr": result.stderr[-8000:], + } + + +def _wait_for_health_urls(urls: Iterable[str], *, timeout_seconds: float, interval_seconds: float) -> dict[str, object]: + results: list[dict[str, object]] = [] + ok = True + errors: list[str] = [] + for url in urls: + health = _wait_for_health_url( + url, + timeout_seconds=timeout_seconds, + interval_seconds=interval_seconds, + ) + health["url"] = url + results.append(health) + if not health.get("ok"): + ok = False + errors.append(f"{url}: {health.get('error') or health.get('status') or 'failed'}") + payload: dict[str, object] = {"ok": ok, "checks": results} + if errors: + payload["error"] = "; ".join(errors) + return payload + + +def _run_restart_command_legacy(command: str | None) -> dict[str, object] | None: + if not command: + return None + return _run_restart_command(command) + + +def _planned_python_install_packages(record: Mapping[str, object]) -> tuple[str, ...]: + raw_plan = record.get("plan") + if not isinstance(raw_plan, list): + return () + packages: list[str] = [] + for item in raw_plan: + if not isinstance(item, Mapping): + continue + if item.get("action") != "install": + continue + package = item.get("python_package") + if isinstance(package, str) and package.strip(): + packages.append(package.strip()) + continue + python_ref = item.get("python_ref") + if isinstance(python_ref, str): + package_name = _python_package_name_from_ref(python_ref) + if package_name: + packages.append(package_name) + return tuple(dict.fromkeys(packages)) + + +def _python_package_name_from_ref(value: str) -> str | None: + cleaned = value.strip() + if not cleaned: + return None + if " @ " in cleaned: + cleaned = cleaned.split(" @ ", 1)[0].strip() + if "==" in cleaned: + cleaned = cleaned.split("==", 1)[0].strip() + if "[" in cleaned: + cleaned = cleaned.split("[", 1)[0].strip() + if re.match(r"^[A-Za-z0-9_.-]+$", cleaned): + return cleaned + return None + + +def _wait_for_health_url(url: str, *, timeout_seconds: float, interval_seconds: float) -> dict[str, object]: + deadline = time.monotonic() + max(timeout_seconds, 0.1) + attempts = 0 + last_error = "" + while time.monotonic() < deadline: + attempts += 1 + try: + with urllib.request.urlopen(url, timeout=min(max(interval_seconds, 0.5), 10.0)) as response: + status = int(getattr(response, "status", 0)) + if 200 <= status < 400: + return {"ok": True, "status": status, "attempts": attempts} + last_error = f"HTTP {status}" + except (OSError, urllib.error.URLError) as exc: + last_error = str(exc) + time.sleep(max(interval_seconds, 0.1)) + return {"ok": False, "attempts": attempts, "error": last_error or "health timeout"} + + +def _update_run_record(path: Path, patch: Mapping[str, object]) -> None: + try: + record = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return + if not isinstance(record, dict): + return + record.update(patch) + _write_json(path, record) + + +def _read_run_record(path: Path) -> dict[str, object] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _run_summary(path: Path, record: Mapping[str, object]) -> dict[str, object]: + supervisor = record.get("supervisor") if isinstance(record.get("supervisor"), Mapping) else {} + commands = record.get("commands") if isinstance(record.get("commands"), list) else [] + plan = record.get("plan") if isinstance(record.get("plan"), list) else [] + return { + "run_id": str(record.get("run_id") or path.parent.name), + "status": str(record.get("status") or "unknown"), + "started_at": record.get("started_at"), + "finished_at": record.get("finished_at"), + "rollback_status": record.get("rollback_status"), + "supervisor_status": supervisor.get("status") if isinstance(supervisor, Mapping) else None, + "error": record.get("error"), + "record_path": str(path), + "commands_count": len(commands), + "planned_modules": [str(item.get("module_id")) for item in plan if isinstance(item, Mapping) and item.get("module_id")], + } + + +def _database_restore_was_applied(record: Mapping[str, object]) -> bool: + restore = record.get("rollback_database_restore") + return isinstance(restore, Mapping) and restore.get("restored") is True + + +def _looks_pinned_dependency_ref(value: str) -> bool: + cleaned = value.strip() + if "==" in cleaned: + return True + if re.match(r"^\d+\.\d+\.\d+(?:[-+][A-Za-z0-9_.-]+)?$", cleaned): + return True + if cleaned.startswith(("git+", "ssh://", "https://", "http://")): + return "#v" in cleaned or "@v" in cleaned or "#refs/tags/" in cleaned or "@refs/tags/" in cleaned + if " @ git+" in cleaned: + return "@v" in cleaned or "@refs/tags/" in cleaned + return False + + +def _snapshot_environment( + run_dir: Path, + *, + webui_root: Path | None, + database_url: str | None, + include_database: bool, + database_backup_command: str | None, + database_restore_command: str | None, + database_restore_check_command: str | None, +) -> dict[str, object]: + snapshot: dict[str, object] = {} + freeze_result = subprocess.run([sys.executable, "-m", "pip", "freeze"], text=True, capture_output=True, check=False) + if freeze_result.returncode == 0: + freeze_path = run_dir / "pip-freeze.txt" + freeze_path.write_text(freeze_result.stdout, encoding="utf-8") + snapshot["pip_freeze"] = freeze_path.name + if include_database: + snapshot["database_backup"] = _snapshot_database( + run_dir, + database_url, + backup_command=database_backup_command, + restore_command=database_restore_command, + restore_check_command=database_restore_check_command, + ) + if webui_root is not None: + snapshot["webui_root"] = str(webui_root) + _snapshot_file(webui_root / "package.json", run_dir / "package.json.before", snapshot, "package_json") + _snapshot_file(webui_root / "package-lock.json", run_dir / "package-lock.json.before", snapshot, "package_lock") + return snapshot + + +def _snapshot_database( + run_dir: Path, + database_url: str | None, + *, + backup_command: str | None, + restore_command: str | None, + restore_check_command: str | None, +) -> dict[str, object]: + if backup_command: + if not restore_command: + raise ModuleInstallerError("--database-backup-command requires --database-restore-command so rollback can restore the snapshot.") + return _snapshot_external_database( + run_dir, + database_url, + backup_command=backup_command, + restore_command=restore_command, + restore_check_command=restore_check_command, + ) + if _sqlite_database_path(database_url) is not None: + return _snapshot_sqlite_database(run_dir, database_url) + raise ModuleInstallerError( + "Automatic database backup for --migrate supports sqlite:/// URLs only. " + "Provide --database-backup-command and --database-restore-command for this database." + ) + + +def _snapshot_sqlite_database(run_dir: Path, database_url: str | None) -> dict[str, object]: + db_path = _sqlite_database_path(database_url) + if db_path is None: + raise ModuleInstallerError("Automatic database backup for --migrate currently supports sqlite:/// URLs only.") + backup_path = run_dir / "database.sqlite.before" + if db_path.exists(): + with sqlite3.connect(str(db_path)) as source, sqlite3.connect(str(backup_path)) as target: + source.backup(target) + else: + backup_path.touch() + return { + "type": "sqlite", + "source": str(db_path), + "path": backup_path.name, + } + + +def _snapshot_external_database( + run_dir: Path, + database_url: str | None, + *, + backup_command: str, + restore_command: str, + restore_check_command: str | None, +) -> dict[str, object]: + backup_path = run_dir / "database.external.backup" + metadata_path = run_dir / "database-backup-metadata.json" + result = _run_database_hook( + backup_command, + run_dir=run_dir, + database_url=database_url, + backup_path=backup_path, + metadata_path=metadata_path, + ) + payload: dict[str, object] = { + "type": "external", + "database_url": database_url or "", + "backup_command": backup_command, + "restore_command": restore_command, + "restore_check_command": restore_check_command, + "backup_path": backup_path.name, + "metadata_path": metadata_path.name, + "return_code": result.returncode, + "stdout": result.stdout[-8000:], + "stderr": result.stderr[-8000:], + } + if result.returncode != 0: + raise ModuleInstallerError(f"Database backup command failed ({result.returncode}): {backup_command}") + if restore_check_command: + restore_check = _run_database_hook( + restore_check_command, + run_dir=run_dir, + database_url=database_url, + backup_path=backup_path, + metadata_path=metadata_path, + ) + payload["restore_check"] = { + "command": restore_check_command, + "return_code": restore_check.returncode, + "stdout": restore_check.stdout[-8000:], + "stderr": restore_check.stderr[-8000:], + } + if restore_check.returncode != 0: + raise ModuleInstallerError(f"Database restore-check command failed ({restore_check.returncode}): {restore_check_command}") + if metadata_path.exists(): + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if isinstance(metadata, dict): + payload["metadata"] = metadata + except json.JSONDecodeError as exc: + raise ModuleInstallerError(f"Database backup metadata is not valid JSON: {metadata_path}") from exc + return payload + + +def _restore_database_snapshot( + run_dir: Path, + snapshot: Mapping[str, object], + *, + database_restore_command: str | None = None, + database_url: str | None = None, +) -> dict[str, object] | None: + raw = snapshot.get("database_backup") + if not isinstance(raw, Mapping): + return None + if raw.get("type") == "external": + return _restore_external_database_snapshot( + run_dir, + raw, + database_restore_command=database_restore_command, + database_url=database_url, + ) + if raw.get("type") != "sqlite": + return {"restored": False, "error": f"Unsupported database backup type: {raw.get('type')}"} + source_name = raw.get("path") + target_name = raw.get("source") + if not isinstance(source_name, str) or not isinstance(target_name, str): + return {"restored": False, "error": "Database backup metadata is incomplete."} + backup_path = run_dir / source_name + target_path = Path(target_name) + if not backup_path.exists(): + return {"restored": False, "error": f"Database backup does not exist: {backup_path}"} + target_path.parent.mkdir(parents=True, exist_ok=True) + if backup_path.stat().st_size == 0: + target_path.unlink(missing_ok=True) + return {"restored": True, "type": "sqlite", "source": str(target_path), "empty": True} + with sqlite3.connect(str(backup_path)) as source, sqlite3.connect(str(target_path)) as target: + source.backup(target) + return {"restored": True, "type": "sqlite", "source": str(target_path)} + + +def _restore_external_database_snapshot( + run_dir: Path, + raw: Mapping[str, object], + *, + database_restore_command: str | None, + database_url: str | None, +) -> dict[str, object]: + command = database_restore_command or raw.get("restore_command") + if not isinstance(command, str) or not command.strip(): + return {"restored": False, "type": "external", "error": "No database restore command is available for this run."} + backup_name = raw.get("backup_path") + metadata_name = raw.get("metadata_path") + raw_database_url = raw.get("database_url") + backup_path = run_dir / backup_name if isinstance(backup_name, str) else run_dir / "database.external.backup" + metadata_path = run_dir / metadata_name if isinstance(metadata_name, str) else run_dir / "database-backup-metadata.json" + result = _run_database_hook( + command, + run_dir=run_dir, + database_url=database_url or (raw_database_url if isinstance(raw_database_url, str) else None), + backup_path=backup_path, + metadata_path=metadata_path, + ) + payload: dict[str, object] = { + "restored": result.returncode == 0, + "type": "external", + "command": command, + "return_code": result.returncode, + "stdout": result.stdout[-8000:], + "stderr": result.stderr[-8000:], + } + if result.returncode != 0: + payload["error"] = f"Database restore command failed ({result.returncode})." + return payload + + +def _run_database_hook( + command: str, + *, + run_dir: Path, + database_url: str | None, + backup_path: Path, + metadata_path: Path, +) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.update({ + "GOVOPLAN_INSTALLER_RUN_DIR": str(run_dir), + "GOVOPLAN_DATABASE_BACKUP_PATH": str(backup_path), + "GOVOPLAN_DATABASE_BACKUP_METADATA": str(metadata_path), + }) + if database_url: + env["GOVOPLAN_DATABASE_URL"] = database_url + env.setdefault("DATABASE_URL", database_url) + return subprocess.run( + command, + cwd=run_dir, + env=env, + shell=True, + text=True, + capture_output=True, + check=False, + ) + + +def _sqlite_database_path(database_url: str | None) -> Path | None: + if not database_url or not database_url.startswith("sqlite:///"): + return None + raw_path = database_url[len("sqlite:///"):] + if raw_path in {"", ":memory:"}: + return None + return Path(raw_path).expanduser().resolve() + + +def _snapshot_file(source: Path, target: Path, snapshot: dict[str, object], key: str) -> None: + if not source.exists(): + return + shutil.copy2(source, target) + snapshot[key] = target.name + + +def _restore_snapshot_file(run_dir: Path, snapshot: Mapping[str, object], key: str, target: Path) -> None: + value = snapshot.get(key) + if not isinstance(value, str): + return + source = run_dir / value + if not source.exists(): + return + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def _mark_applied(item: ModuleInstallPlanItem) -> ModuleInstallPlanItem: + if item.status != "planned": + return item + return ModuleInstallPlanItem( + module_id=item.module_id, + action=item.action, + python_package=item.python_package, + python_ref=item.python_ref, + webui_package=item.webui_package, + webui_ref=item.webui_ref, + destroy_data=item.destroy_data, + status="applied", + notes=item.notes, + ) + + +def _structured_command(argv: list[str], *, cwd: Path | None = None) -> dict[str, Any]: + return { + "argv": argv, + "cwd": str(cwd) if cwd is not None else None, + "display": _display_command(argv, cwd=cwd), + } + + +def _display_command(argv: list[str], *, cwd: Path | None = None) -> str: + command = " ".join(_quote_arg(arg) for arg in argv) + if cwd is not None: + return f"cd {_quote_arg(str(cwd))} && {command}" + return command + + +def _quote_arg(value: str) -> str: + import shlex + + return shlex.quote(value) + + +def _command_record(command: Mapping[str, Any]) -> dict[str, object]: + payload: dict[str, object] = { + "display": str(command["display"]), + "argv": [str(item) for item in command["argv"]], + } + if command.get("cwd"): + payload["cwd"] = str(command["cwd"]) + return payload + + +class _InstallerLock(AbstractContextManager[None]): + def __init__(self, lock_path: Path) -> None: + self.lock_path = lock_path + self.fd: int | None = None + + def __enter__(self) -> None: + self.lock_path.parent.mkdir(parents=True, exist_ok=True) + try: + self.fd = os.open(self.lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError as exc: + raise ModuleInstallerError(f"Module installer lock already exists: {self.lock_path}") from exc + os.write(self.fd, json.dumps({"pid": os.getpid(), "created_at": datetime.now(tz=UTC).isoformat()}).encode("utf-8")) + return None + + def __exit__(self, exc_type, exc, traceback) -> bool | None: + if self.fd is not None: + os.close(self.fd) + self.lock_path.unlink(missing_ok=True) + return None + + +def _installer_lock(runtime_dir: Path) -> _InstallerLock: + return _InstallerLock(runtime_dir / "install.lock") + + +def _request_queue_lock(runtime_dir: Path) -> _InstallerLock: + return _InstallerLock(runtime_dir / "requests.lock") + + +def _daemon_status_path(runtime_dir: Path) -> Path: + return runtime_dir / "daemon.status.json" + + +def _request_record_path(runtime_dir: Path, request_id: str) -> Path: + return runtime_dir / "requests" / f"{request_id}.json" + + +def _run_id() -> str: + return datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%S%fZ") + + +def _write_json(path: Path, payload: Mapping[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def _process_is_running(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True diff --git a/src/govoplan_core/core/module_license.py b/src/govoplan_core/core/module_license.py new file mode 100644 index 0000000..c2c0940 --- /dev/null +++ b/src/govoplan_core/core/module_license.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import base64 +import binascii +from datetime import UTC, datetime +import json +import os +from pathlib import Path +from typing import Any +import urllib.error +import urllib.request + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + + +def module_license_decision(required_features: list[str] | tuple[str, ...]) -> dict[str, object]: + required = tuple(dict.fromkeys(str(item).strip() for item in required_features if str(item).strip())) + if not required: + return {"allowed": True, "required_features": [], "missing_features": [], "enforced": False} + validation = validate_module_license() + enforced = _license_enforcement_enabled() + if not validation["valid"]: + return { + "allowed": not enforced, + "required_features": list(required), + "missing_features": list(required), + "enforced": enforced, + "license": validation, + "reason": validation["error"] or "No valid license is configured.", + } + licensed_features = set(validation["features"]) + missing = tuple(feature for feature in required if feature not in licensed_features) + return { + "allowed": not missing or not enforced, + "required_features": list(required), + "missing_features": list(missing), + "enforced": enforced, + "license": validation, + "reason": "Missing licensed feature(s): " + ", ".join(missing) if missing else None, + } + + +def validate_module_license( + path: Path | None = None, + *, + trusted_keys: dict[str, str] | None = None, + require_trusted: bool | None = None, +) -> dict[str, object]: + license_path = path or _configured_license_path() + effective_require_trusted = _license_enforcement_enabled() if require_trusted is None else require_trusted + if license_path is None: + return _license_result(valid=not effective_require_trusted, configured=False, path=None, error="No license file configured.") + if not license_path.exists(): + return _license_result(valid=False, configured=True, path=license_path, error=f"License file does not exist: {license_path}") + try: + payload = json.loads(license_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("License file must contain a JSON object.") + features = _string_list(payload.get("features")) + valid_from = _parse_datetime(payload.get("valid_from")) + valid_until = _parse_datetime(payload.get("valid_until")) + now = datetime.now(tz=UTC) + if valid_from is not None and now < valid_from: + return _license_result(valid=False, configured=True, path=license_path, payload=payload, features=features, error=f"License is not valid before {valid_from.isoformat()}.") + if valid_until is not None and now > valid_until: + return _license_result(valid=False, configured=True, path=license_path, payload=payload, features=features, error=f"License expired at {valid_until.isoformat()}.") + signature_state = _license_signature_state(payload, trusted_keys=trusted_keys if trusted_keys is not None else _configured_trusted_keys()) + if signature_state.get("fatal"): + return _license_result(valid=False, configured=True, path=license_path, payload=payload, features=features, signature_state=signature_state, error=str(signature_state["error"])) + if effective_require_trusted and not signature_state["trusted"]: + return _license_result(valid=False, configured=True, path=license_path, payload=payload, features=features, signature_state=signature_state, error=str(signature_state["error"] or "License must be signed by a trusted key.")) + return _license_result(valid=True, configured=True, path=license_path, payload=payload, features=features, signature_state=signature_state) + except Exception as exc: + return _license_result(valid=False, configured=True, path=license_path, error=str(exc)) + + +def _license_result( + *, + valid: bool, + configured: bool, + path: Path | None, + error: str | None = None, + payload: dict[str, object] | None = None, + features: list[str] | None = None, + signature_state: dict[str, object] | None = None, +) -> dict[str, object]: + state = signature_state or {"signed": False, "trusted": False, "key_id": None} + return { + "valid": valid, + "configured": configured, + "path": str(path) if path is not None else None, + "license_id": str(payload.get("license_id")) if payload and payload.get("license_id") else None, + "subject": str(payload.get("subject")) if payload and payload.get("subject") else None, + "features": features or [], + "valid_from": str(payload.get("valid_from")) if payload and payload.get("valid_from") else None, + "valid_until": str(payload.get("valid_until")) if payload and payload.get("valid_until") else None, + "signed": bool(state.get("signed")), + "trusted": bool(state.get("trusted")), + "key_id": state.get("key_id") if isinstance(state.get("key_id"), str) else None, + "error": error, + } + + +def _configured_license_path() -> Path | None: + value = os.getenv("GOVOPLAN_LICENSE_FILE", "").strip() + return Path(value).expanduser() if value else None + + +def _license_enforcement_enabled() -> bool: + return os.getenv("GOVOPLAN_LICENSE_ENFORCEMENT", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _configured_trusted_keys() -> dict[str, str]: + file_value = os.getenv("GOVOPLAN_LICENSE_TRUSTED_KEYS_FILE", "").strip() + if file_value: + path = Path(file_value).expanduser() + if not path.exists(): + raise ValueError(f"Trusted license key file does not exist: {path}") + return _parse_trusted_keys(path.read_text(encoding="utf-8")) + url_value = os.getenv("GOVOPLAN_LICENSE_TRUSTED_KEYS_URL", "").strip() + if url_value: + return _parse_trusted_keys(_read_trusted_keys_url(url_value)) + value = os.getenv("GOVOPLAN_LICENSE_TRUSTED_KEYS", "").strip() + return _parse_trusted_keys(value) if value else {} + + +def _configured_trusted_keys_cache_path() -> Path | None: + value = os.getenv("GOVOPLAN_LICENSE_TRUSTED_KEYS_CACHE", "").strip() + return Path(value).expanduser() if value else None + + +def _read_trusted_keys_url(url: str) -> str: + if not url.startswith(("https://", "http://")): + raise ValueError("Trusted license key URL must use http:// or https://.") + cache_path = _configured_trusted_keys_cache_path() + try: + with urllib.request.urlopen(url, timeout=15) as response: + body = response.read().decode("utf-8") + if cache_path is not None: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body, encoding="utf-8") + return body + except (OSError, urllib.error.URLError): + if cache_path is not None and cache_path.exists(): + return cache_path.read_text(encoding="utf-8") + raise + + +def _parse_trusted_keys(value: str) -> dict[str, str]: + payload = json.loads(value) + if isinstance(payload, dict) and isinstance(payload.get("keys"), list): + keys: dict[str, str] = {} + now = datetime.now(tz=UTC) + for item in payload["keys"]: + if not isinstance(item, dict): + continue + if str(item.get("status") or "active").strip().lower() in {"revoked", "disabled", "retired"}: + continue + not_before = _parse_datetime(item.get("not_before")) + not_after = _parse_datetime(item.get("not_after")) + if not_before is not None and now < not_before: + continue + if not_after is not None and now > not_after: + continue + key_id = str(item.get("key_id") or "").strip() + public_key = str(item.get("public_key") or item.get("public_key_base64") or "").strip() + if key_id and public_key: + keys[key_id] = public_key + return keys + if not isinstance(payload, dict): + raise ValueError("Trusted license keys must be a JSON object.") + return {str(key): str(item) for key, item in payload.items() if str(key).strip() and str(item).strip()} + + +def _license_signature_state(payload: dict[str, object], *, trusted_keys: dict[str, str]) -> dict[str, object]: + signature = payload.get("signature") + if signature is None: + return {"signed": False, "trusted": False, "key_id": None, "error": "License is unsigned.", "fatal": False} + if not isinstance(signature, dict): + return {"signed": True, "trusted": False, "key_id": None, "error": "License signature must be an object.", "fatal": True} + algorithm = str(signature.get("algorithm") or "").strip().lower() + key_id = str(signature.get("key_id") or "").strip() + value = str(signature.get("value") or "").strip() + if algorithm != "ed25519": + return {"signed": True, "trusted": False, "key_id": key_id or None, "error": f"Unsupported license signature algorithm: {algorithm!r}", "fatal": True} + if not key_id or not value: + return {"signed": True, "trusted": False, "key_id": key_id or None, "error": "License signature requires key_id and value.", "fatal": True} + trusted_key = trusted_keys.get(key_id) + if not trusted_key: + return {"signed": True, "trusted": False, "key_id": key_id, "error": f"License signature key is not trusted: {key_id}", "fatal": False} + try: + public_key = Ed25519PublicKey.from_public_bytes(base64.b64decode(trusted_key, validate=True)) + signed_payload = dict(payload) + signed_payload.pop("signature", None) + public_key.verify(base64.b64decode(value, validate=True), _canonical_bytes(signed_payload)) + except (ValueError, binascii.Error, InvalidSignature) as exc: + return {"signed": True, "trusted": False, "key_id": key_id, "error": f"License signature verification failed: {exc}", "fatal": True} + return {"signed": True, "trusted": True, "key_id": key_id, "error": None, "fatal": False} + + +def _canonical_bytes(payload: object) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _parse_datetime(value: object) -> datetime | None: + if value is None: + return None + text = str(value).strip() + if not text: + return None + if text.endswith("Z"): + text = text[:-1] + "+00:00" + parsed = datetime.fromisoformat(text) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _string_list(value: object) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("License features must be a list.") + return [str(item).strip() for item in value if str(item).strip()] diff --git a/src/govoplan_core/core/module_management.py b/src/govoplan_core/core/module_management.py new file mode 100644 index 0000000..e34cfdd --- /dev/null +++ b/src/govoplan_core/core/module_management.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +import shlex + +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from govoplan_core.admin.models import SystemSettings +from govoplan_core.admin.settings import SYSTEM_SETTINGS_ID, get_system_settings +from govoplan_core.core.discovery import iter_module_entry_points +from govoplan_core.core.modules import ModuleManifest +from govoplan_core.db.session import get_database +from govoplan_core.server.registry import parse_enabled_modules + +MODULE_SETTINGS_KEY = "module_management" +INSTALL_PLAN_KEY = "install_plan" +REQUIRED_PLATFORM_MODULES = ("tenancy", "access") +PROTECTED_MODULES = (*REQUIRED_PLATFORM_MODULES, "admin") +INSTALL_PLAN_ACTIONS = ("install", "uninstall") +INSTALL_PLAN_STATUSES = ("planned", "applied", "blocked") +LOCAL_DEPENDENCY_REF_PREFIXES = ("file:", "path:", "workspace:", "link:") + + +class ModuleManagementError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class ModuleStatePlan: + enabled_modules: tuple[str, ...] + added_dependencies: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class ModuleInstallPlanItem: + module_id: str + action: str + python_package: str | None = None + python_ref: str | None = None + webui_package: str | None = None + webui_ref: str | None = None + destroy_data: bool = False + status: str = "planned" + notes: str | None = None + + def as_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "module_id": self.module_id, + "action": self.action, + "status": self.status, + } + if self.destroy_data: + payload["destroy_data"] = True + for key in ("python_package", "python_ref", "webui_package", "webui_ref", "notes"): + value = getattr(self, key) + if value: + payload[key] = value + return payload + + +@dataclass(frozen=True, slots=True) +class ModuleInstallPlan: + items: tuple[ModuleInstallPlanItem, ...] = () + updated_at: str | None = None + + +def configured_enabled_modules(value: str | Iterable[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(parse_enabled_modules(value))) + + +def startup_candidate_module_ids( + configured: str | Iterable[str], + desired: Iterable[str] | None = None, +) -> tuple[str, ...]: + fallback = configured_enabled_modules(configured) + candidates = [*fallback] + if desired is not None: + candidates.extend(str(item).strip() for item in desired if str(item).strip()) + candidates.extend(REQUIRED_PLATFORM_MODULES) + if "admin" in fallback: + candidates.append("admin") + return tuple(dict.fromkeys(candidates)) + + +def saved_desired_enabled_modules(session: Session, fallback: Iterable[str]) -> tuple[str, ...]: + item = session.get(SystemSettings, SYSTEM_SETTINGS_ID) + if item is None: + return tuple(fallback) + return desired_enabled_modules_from_settings(item.settings or {}, fallback) + + +def desired_enabled_modules_from_settings(settings: Mapping[str, object], fallback: Iterable[str]) -> tuple[str, ...]: + raw_management = settings.get(MODULE_SETTINGS_KEY) + if not isinstance(raw_management, Mapping): + return tuple(fallback) + raw_enabled = raw_management.get("desired_enabled") + if not isinstance(raw_enabled, list | tuple): + return tuple(fallback) + normalized = [str(item).strip() for item in raw_enabled if str(item).strip()] + return tuple(dict.fromkeys(normalized)) or tuple(fallback) + + +def load_startup_enabled_modules( + configured: str | Iterable[str], + *, + available: Mapping[str, ModuleManifest] | None = None, +) -> tuple[str, ...]: + fallback = configured_enabled_modules(configured) + try: + with get_database().session() as session: + desired = saved_desired_enabled_modules(session, fallback) + except (RuntimeError, SQLAlchemyError): + return fallback + if available is None: + return desired + + protected = list(REQUIRED_PLATFORM_MODULES) + if "admin" in fallback: + protected.append("admin") + protected = [module_id for module_id in protected if module_id in available] + requested = [module_id for module_id in desired if module_id in available] + if not requested: + requested = [module_id for module_id in fallback if module_id in available] + return plan_desired_enabled_modules(requested, available, protected_modules=protected).enabled_modules + + +def save_desired_enabled_modules(session: Session, enabled_modules: Iterable[str]) -> tuple[str, ...]: + item = get_system_settings(session) + settings = dict(item.settings or {}) + management = _management_settings(settings) + management["desired_enabled"] = list(dict.fromkeys(enabled_modules)) + settings[MODULE_SETTINGS_KEY] = management + item.settings = settings + session.add(item) + session.flush() + return tuple(management["desired_enabled"]) # type: ignore[arg-type] + + +def module_install_plan_from_settings(settings: Mapping[str, object]) -> ModuleInstallPlan: + raw_management = settings.get(MODULE_SETTINGS_KEY) + if not isinstance(raw_management, Mapping): + return ModuleInstallPlan() + raw_plan = raw_management.get(INSTALL_PLAN_KEY) + if not isinstance(raw_plan, Mapping): + return ModuleInstallPlan() + raw_items = raw_plan.get("items") + if not isinstance(raw_items, list | tuple): + return ModuleInstallPlan(updated_at=_clean_optional_string(raw_plan.get("updated_at"))) + return ModuleInstallPlan( + items=tuple(normalize_module_install_plan_item(item) for item in raw_items), + updated_at=_clean_optional_string(raw_plan.get("updated_at")), + ) + + +def saved_module_install_plan(session: Session) -> ModuleInstallPlan: + item = session.get(SystemSettings, SYSTEM_SETTINGS_ID) + if item is None: + return ModuleInstallPlan() + return module_install_plan_from_settings(item.settings or {}) + + +def save_module_install_plan( + session: Session, + items: Iterable[Mapping[str, object] | ModuleInstallPlanItem], +) -> ModuleInstallPlan: + normalized = tuple(normalize_module_install_plan_item(item) for item in items) + item = get_system_settings(session) + settings = dict(item.settings or {}) + management = _management_settings(settings) + updated_at = datetime.now(tz=UTC).isoformat() + management[INSTALL_PLAN_KEY] = { + "updated_at": updated_at, + "items": [entry.as_dict() for entry in normalized], + } + settings[MODULE_SETTINGS_KEY] = management + item.settings = settings + session.add(item) + session.flush() + return ModuleInstallPlan(items=normalized, updated_at=updated_at) + + +def clear_module_install_plan(session: Session) -> ModuleInstallPlan: + return save_module_install_plan(session, ()) + + +def module_install_plan_commands( + plan: ModuleInstallPlan | Iterable[ModuleInstallPlanItem], + *, + statuses: Iterable[str] = ("planned",), +) -> tuple[str, ...]: + items = plan.items if isinstance(plan, ModuleInstallPlan) else tuple(plan) + included_statuses = {status for status in statuses} + commands: list[str] = [] + for item in items: + if item.status not in included_statuses: + continue + if item.action == "install": + if item.python_ref: + commands.append(f"python -m pip install {shlex.quote(item.python_ref)}") + if item.webui_package and item.webui_ref: + commands.append( + "npm pkg set " + + shlex.quote(f"dependencies.{item.webui_package}={item.webui_ref}") + + " && npm install" + ) + elif item.action == "uninstall": + if item.python_package: + commands.append(f"python -m pip uninstall -y {shlex.quote(item.python_package)}") + if item.webui_package: + commands.append( + "npm pkg delete " + + shlex.quote(f"dependencies.{item.webui_package}") + + " && npm install" + ) + return tuple(commands) + + +def installed_module_distribution_name(module_id: str) -> str | None: + clean_module_id = str(module_id).strip() + if not clean_module_id: + return None + for entry_point in iter_module_entry_points(): + if entry_point.name != clean_module_id: + continue + distribution = getattr(entry_point, "dist", None) + metadata = getattr(distribution, "metadata", None) + if metadata is None: + return None + name = metadata.get("Name") + return str(name).strip() if name else None + return None + + +def module_uninstall_plan_item( + module_id: str, + available: Mapping[str, ModuleManifest], +) -> ModuleInstallPlanItem: + clean_module_id = str(module_id).strip() + if not clean_module_id: + raise ModuleManagementError("Uninstall plan needs a module id.") + manifest = available.get(clean_module_id) + if manifest is None: + raise ModuleManagementError(f"Module {clean_module_id!r} is not installed.") + python_package = installed_module_distribution_name(clean_module_id) + if not python_package: + raise ModuleManagementError( + f"Could not determine the installed Python distribution for module {clean_module_id!r}." + ) + frontend = manifest.frontend + return ModuleInstallPlanItem( + module_id=clean_module_id, + action="uninstall", + python_package=python_package, + webui_package=frontend.package_name if frontend and frontend.package_name else None, + notes="Non-destructive uninstall leaves module data and schema dormant.", + ) + + +def desired_modules_after_package_plan( + desired_enabled: Iterable[str], + plan: ModuleInstallPlan, + *, + activate_installed_modules: bool = True, + remove_uninstalled_modules_from_desired: bool = True, +) -> tuple[str, ...]: + desired = [str(item).strip() for item in desired_enabled if str(item).strip()] + planned_items = tuple(item for item in plan.items if item.status == "planned") + if remove_uninstalled_modules_from_desired: + removed = {item.module_id for item in planned_items if item.action == "uninstall"} + desired = [module_id for module_id in desired if module_id not in removed] + if activate_installed_modules: + desired.extend(item.module_id for item in planned_items if item.action == "install") + return tuple(dict.fromkeys(desired)) + + +def normalize_module_install_plan_item( + item: Mapping[str, object] | ModuleInstallPlanItem, +) -> ModuleInstallPlanItem: + if isinstance(item, ModuleInstallPlanItem): + raw = item.as_dict() + elif isinstance(item, Mapping): + raw = item + else: + raise ModuleManagementError("Install plan item must be an object.") + + module_id = _required_string(raw.get("module_id"), "module_id") + action = _required_string(raw.get("action"), "action") + status = _clean_optional_string(raw.get("status")) or "planned" + python_package = _clean_optional_string(raw.get("python_package")) + python_ref = _clean_optional_string(raw.get("python_ref")) + webui_package = _clean_optional_string(raw.get("webui_package")) + webui_ref = _clean_optional_string(raw.get("webui_ref")) + destroy_data = _clean_bool(raw.get("destroy_data")) + notes = _clean_optional_string(raw.get("notes")) + + if action not in INSTALL_PLAN_ACTIONS: + raise ModuleManagementError(f"Unsupported install plan action for {module_id!r}: {action!r}.") + if status not in INSTALL_PLAN_STATUSES: + raise ModuleManagementError(f"Unsupported install plan status for {module_id!r}: {status!r}.") + if action == "install" and not python_ref: + raise ModuleManagementError(f"Install plan item {module_id!r} needs a Python package reference.") + if action == "uninstall" and not python_package: + raise ModuleManagementError(f"Uninstall plan item {module_id!r} needs a Python package name.") + if action != "uninstall" and destroy_data: + raise ModuleManagementError(f"Install plan item {module_id!r} can only destroy data during uninstall.") + if action == "install" and bool(webui_package) != bool(webui_ref): + raise ModuleManagementError(f"Install plan item {module_id!r} needs both WebUI package and WebUI reference, or neither.") + if action == "uninstall" and webui_ref and not webui_package: + raise ModuleManagementError(f"Uninstall plan item {module_id!r} has a WebUI reference but no WebUI package.") + if python_ref: + _validate_dependency_ref(python_ref, field="python_ref", module_id=module_id) + if webui_ref: + _validate_dependency_ref(webui_ref, field="webui_ref", module_id=module_id) + + return ModuleInstallPlanItem( + module_id=module_id, + action=action, + python_package=python_package, + python_ref=python_ref, + webui_package=webui_package, + webui_ref=webui_ref, + destroy_data=destroy_data, + status=status, + notes=notes, + ) + + +def plan_desired_enabled_modules( + requested_enabled: Iterable[str], + available: Mapping[str, ModuleManifest], + *, + protected_modules: Iterable[str] = PROTECTED_MODULES, +) -> ModuleStatePlan: + requested = {str(item).strip() for item in requested_enabled if str(item).strip()} + protected = {str(item).strip() for item in protected_modules if str(item).strip()} + requested.update(protected) + missing = sorted(module_id for module_id in requested if module_id not in available) + if missing: + raise ModuleManagementError("Unknown or uninstalled modules: " + ", ".join(missing)) + + added_dependencies: set[str] = set() + visiting: set[str] = set() + visited: set[str] = set() + ordered: list[str] = [] + + def visit(module_id: str) -> None: + if module_id in visited: + return + if module_id in visiting: + raise ModuleManagementError(f"Module dependency cycle includes {module_id!r}.") + manifest = available.get(module_id) + if manifest is None: + raise ModuleManagementError(f"Module {module_id!r} is required but not installed.") + visiting.add(module_id) + for dependency_id in manifest.dependencies: + if dependency_id not in available: + raise ModuleManagementError(f"Module {module_id!r} depends on uninstalled module {dependency_id!r}.") + if dependency_id not in requested: + added_dependencies.add(dependency_id) + requested.add(dependency_id) + visit(dependency_id) + visiting.remove(module_id) + visited.add(module_id) + ordered.append(module_id) + + for module_id in sorted(requested): + visit(module_id) + return ModuleStatePlan( + enabled_modules=tuple(dict.fromkeys(ordered)), + added_dependencies=tuple(sorted(added_dependencies)), + ) + + +def module_dependents(available: Mapping[str, ModuleManifest]) -> dict[str, tuple[str, ...]]: + dependents: dict[str, list[str]] = {module_id: [] for module_id in available} + for manifest in available.values(): + for dependency_id in manifest.dependencies: + dependents.setdefault(dependency_id, []).append(manifest.id) + return {module_id: tuple(sorted(items)) for module_id, items in dependents.items()} + + +def _management_settings(settings: Mapping[str, object]) -> dict[str, object]: + raw_management = settings.get(MODULE_SETTINGS_KEY) + if isinstance(raw_management, Mapping): + return dict(raw_management) + return {} + + +def _required_string(value: object, field: str) -> str: + cleaned = _clean_optional_string(value) + if not cleaned: + raise ModuleManagementError(f"Install plan item needs {field}.") + return cleaned + + +def _clean_optional_string(value: object) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None + + +def _clean_bool(value: object) -> bool: + if isinstance(value, bool): + return value + if value is None: + return False + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + +def _validate_dependency_ref(value: str, *, field: str, module_id: str) -> None: + lowered = value.lower() + if lowered.startswith(LOCAL_DEPENDENCY_REF_PREFIXES) or value.startswith(("/", "./", "../", "~")): + raise ModuleManagementError( + f"Install plan item {module_id!r} uses a local {field}; use a tagged package or git ref instead." + ) diff --git a/src/govoplan_core/core/module_package_catalog.py b/src/govoplan_core/core/module_package_catalog.py new file mode 100644 index 0000000..dcca99e --- /dev/null +++ b/src/govoplan_core/core/module_package_catalog.py @@ -0,0 +1,651 @@ +from __future__ import annotations + +import base64 +import binascii +from datetime import UTC, datetime +from pathlib import Path +import json +import os +from typing import Any +import urllib.error +import urllib.request + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey + + +def module_package_catalog( + path: Path | str | None = None, + *, + require_trusted: bool | None = None, + approved_channels: tuple[str, ...] | None = None, + trusted_keys: dict[str, str] | None = None, +) -> tuple[dict[str, object], ...]: + catalog_source = _catalog_source(path) + if catalog_source is None or not _catalog_source_exists(catalog_source): + return () + validation = validate_module_package_catalog( + catalog_source, + require_trusted=require_trusted, + approved_channels=approved_channels, + trusted_keys=trusted_keys, + ) + if not validation["valid"]: + raise ValueError(str(validation["error"] or "Module package catalog is invalid.")) + return tuple(item for item in validation["modules"] if isinstance(item, dict)) + + +def validate_module_package_catalog( + path: Path | str | None = None, + *, + require_trusted: bool | None = None, + approved_channels: tuple[str, ...] | None = None, + trusted_keys: dict[str, str] | None = None, +) -> dict[str, object]: + catalog_source = _catalog_source(path) + effective_require_trusted = _configured_require_signature() if require_trusted is None else require_trusted + effective_approved_channels = _configured_approved_channels() if approved_channels is None else approved_channels + effective_trusted_keys = trusted_keys if trusted_keys is not None else _configured_trusted_keys() + if catalog_source is not None and not _catalog_source_exists(catalog_source): + return { + "valid": False, + "configured": True, + "path": str(catalog_source), + "source": str(catalog_source), + "source_type": _catalog_source_type(catalog_source), + "modules": [], + "channel": None, + "sequence": None, + "generated_at": None, + "expires_at": None, + "signed": False, + "trusted": False, + "key_id": None, + "warnings": [], + "error": f"Module package catalog does not exist: {catalog_source}", + } + warnings: list[str] = [] + try: + payload = _read_catalog_payload(catalog_source) + modules = _normalize_catalog_modules(payload) + channel = _catalog_channel(payload) + sequence = _catalog_sequence(payload) + generated_at = _catalog_optional_text(payload, "generated_at") + expires_at = _catalog_optional_text(payload, "expires_at") + signature_state = _catalog_signature_state(payload, trusted_keys=effective_trusted_keys) + freshness = _catalog_freshness_state(payload) + replay = _catalog_replay_state(channel=channel, sequence=sequence) + except Exception as exc: + return { + "valid": False, + "configured": catalog_source is not None, + "path": str(catalog_source) if catalog_source is not None else None, + "source": str(catalog_source) if catalog_source is not None else None, + "source_type": _catalog_source_type(catalog_source), + "modules": [], + "channel": None, + "sequence": None, + "generated_at": None, + "expires_at": None, + "signed": False, + "trusted": False, + "key_id": None, + "warnings": [], + "error": str(exc), + } + if signature_state.get("fatal"): + return { + "valid": False, + "configured": catalog_source is not None, + "path": str(catalog_source) if catalog_source is not None else None, + "source": str(catalog_source) if catalog_source is not None else None, + "source_type": _catalog_source_type(catalog_source), + "modules": [], + "channel": channel, + "sequence": sequence, + "generated_at": generated_at, + "expires_at": expires_at, + "signed": signature_state["signed"], + "trusted": signature_state["trusted"], + "key_id": signature_state["key_id"], + "warnings": [], + "error": str(signature_state["error"] or "Module package catalog signature is invalid."), + } + if effective_approved_channels and channel not in effective_approved_channels: + return { + "valid": False, + "configured": catalog_source is not None, + "path": str(catalog_source) if catalog_source is not None else None, + "source": str(catalog_source) if catalog_source is not None else None, + "source_type": _catalog_source_type(catalog_source), + "modules": [], + "channel": channel, + "sequence": sequence, + "generated_at": generated_at, + "expires_at": expires_at, + "signed": signature_state["signed"], + "trusted": signature_state["trusted"], + "key_id": signature_state["key_id"], + "warnings": [], + "error": f"Module package catalog channel {channel!r} is not approved. Approved channels: {', '.join(effective_approved_channels)}.", + } + if effective_require_trusted and not signature_state["trusted"]: + return { + "valid": False, + "configured": catalog_source is not None, + "path": str(catalog_source) if catalog_source is not None else None, + "source": str(catalog_source) if catalog_source is not None else None, + "source_type": _catalog_source_type(catalog_source), + "modules": [], + "channel": channel, + "sequence": sequence, + "generated_at": generated_at, + "expires_at": expires_at, + "signed": signature_state["signed"], + "trusted": False, + "key_id": signature_state["key_id"], + "warnings": [], + "error": str(signature_state["error"] or "Module package catalog must be signed by a trusted key."), + } + if not freshness["valid"]: + return _invalid_catalog_result( + catalog_source, + modules=(), + channel=channel, + sequence=sequence, + generated_at=generated_at, + expires_at=expires_at, + signature_state=signature_state, + error=str(freshness["error"]), + ) + if not replay["valid"]: + return _invalid_catalog_result( + catalog_source, + modules=(), + channel=channel, + sequence=sequence, + generated_at=generated_at, + expires_at=expires_at, + signature_state=signature_state, + error=str(replay["error"]), + ) + warnings.extend(str(item) for item in freshness.get("warnings", ()) if item) + warnings.extend(str(item) for item in replay.get("warnings", ()) if item) + if not signature_state["signed"]: + warnings.append("Catalog is unsigned; use only for local development unless signature enforcement is disabled intentionally.") + elif not signature_state["trusted"]: + warnings.append(str(signature_state["error"] or "Catalog signature could not be verified against a trusted key.")) + return { + "valid": True, + "configured": catalog_source is not None and _catalog_source_exists(catalog_source), + "path": str(catalog_source) if catalog_source is not None else None, + "source": str(catalog_source) if catalog_source is not None else None, + "source_type": _catalog_source_type(catalog_source), + "modules": list(modules), + "channel": channel, + "sequence": sequence, + "generated_at": generated_at, + "expires_at": expires_at, + "signed": signature_state["signed"], + "trusted": signature_state["trusted"], + "key_id": signature_state["key_id"], + "warnings": warnings, + "error": None, + } + + +def sign_module_package_catalog( + *, + path: Path, + key_id: str, + private_key_path: Path, + output_path: Path | None = None, +) -> Path: + payload = _read_catalog_payload(path) + if not isinstance(payload, dict): + raise ValueError("Only object-style module package catalogs can be signed.") + signature_payload = dict(payload) + signature_payload.pop("signature", None) + signature_payload.pop("signatures", None) + private_key = _load_private_key(private_key_path) + signature = private_key.sign(_canonical_catalog_bytes(signature_payload)) + signature_payload["signature"] = { + "algorithm": "ed25519", + "key_id": key_id, + "value": base64.b64encode(signature).decode("ascii"), + } + target = output_path or path + target.write_text(json.dumps(signature_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return target + + +def record_module_package_catalog_acceptance(validation: dict[str, object]) -> None: + state_path = _configured_sequence_state_path() + if state_path is None or validation.get("valid") is not True: + return + channel = validation.get("channel") + sequence = validation.get("sequence") + if not isinstance(channel, str) or not isinstance(sequence, int): + return + try: + state = json.loads(state_path.read_text(encoding="utf-8")) if state_path.exists() else {} + except json.JSONDecodeError: + state = {} + if not isinstance(state, dict): + state = {} + channels = state.get("channels") + if not isinstance(channels, dict): + channels = {} + channel_state = channels.get(channel) + if not isinstance(channel_state, dict): + channel_state = {} + channel_state["last_sequence"] = max(int(channel_state.get("last_sequence") or 0), sequence) + channel_state["accepted_at"] = datetime.now(tz=UTC).isoformat().replace("+00:00", "Z") + channel_state["key_id"] = validation.get("key_id") + channel_state["source"] = validation.get("source") or validation.get("path") + channels[channel] = channel_state + state["channels"] = channels + state_path.parent.mkdir(parents=True, exist_ok=True) + state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _catalog_source(path: Path | str | None) -> Path | str | None: + if path is not None: + return path if isinstance(path, str) and _is_http_url(path) else Path(path).expanduser() + url = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL", "").strip() + if url: + return url + return _configured_catalog_path() + + +def _configured_catalog_path() -> Path | None: + value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG", "").strip() + return Path(value).expanduser() if value else None + + +def _configured_catalog_cache_path() -> Path | None: + value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_CACHE", "").strip() + return Path(value).expanduser() if value else None + + +def _configured_sequence_state_path() -> Path | None: + value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE", "").strip() + return Path(value).expanduser() if value else None + + +def _configured_enforce_sequence() -> bool: + return os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _configured_require_signature() -> bool: + return os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _configured_approved_channels() -> tuple[str, ...]: + value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS", "").strip() + if not value: + return () + return tuple(item.strip() for item in value.split(",") if item.strip()) + + +def _configured_trusted_keys() -> dict[str, str]: + file_value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "").strip() + if file_value: + path = Path(file_value).expanduser() + if not path.exists(): + raise ValueError(f"Trusted catalog key file does not exist: {path}") + return _parse_trusted_keys(path.read_text(encoding="utf-8")) + url_value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_URL", "").strip() + if url_value: + return _parse_trusted_keys(_read_trusted_keys_url(url_value)) + value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS", "").strip() + return _parse_trusted_keys(value) if value else {} + + +def _configured_trusted_keys_cache_path() -> Path | None: + value = os.getenv("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_CACHE", "").strip() + return Path(value).expanduser() if value else None + + +def _read_trusted_keys_url(url: str) -> str: + if not _is_http_url(url): + raise ValueError("Trusted catalog key URL must use http:// or https://.") + cache_path = _configured_trusted_keys_cache_path() + try: + with urllib.request.urlopen(url, timeout=15) as response: + body = response.read().decode("utf-8") + if cache_path is not None: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body, encoding="utf-8") + return body + except (OSError, urllib.error.URLError): + if cache_path is not None and cache_path.exists(): + return cache_path.read_text(encoding="utf-8") + raise + + +def _parse_trusted_keys(value: str) -> dict[str, str]: + payload = json.loads(value) + if isinstance(payload, dict) and isinstance(payload.get("keys"), list): + keys: dict[str, str] = {} + now = datetime.now(tz=UTC) + for item in payload["keys"]: + if not isinstance(item, dict): + continue + status = str(item.get("status") or "active").strip().lower() + if status in {"revoked", "disabled", "retired"}: + continue + not_before = _parse_datetime(item.get("not_before")) + not_after = _parse_datetime(item.get("not_after")) + if not_before is not None and now < not_before: + continue + if not_after is not None and now > not_after: + continue + key_id = str(item.get("key_id") or "").strip() + public_key = str(item.get("public_key") or item.get("public_key_base64") or "").strip() + if key_id and public_key: + keys[key_id] = public_key + return keys + if not isinstance(payload, dict): + raise ValueError("Trusted catalog keys must be a JSON object mapping key ids to base64 public keys.") + return {str(key): str(item) for key, item in payload.items() if str(key).strip() and str(item).strip()} + + +def _read_catalog_payload(source: Path | str | None) -> object: + if source is None: + return {"modules": []} + if isinstance(source, str) and _is_http_url(source): + return json.loads(_read_catalog_url(source)) + return json.loads(Path(source).read_text(encoding="utf-8")) + + +def _read_catalog_url(url: str) -> str: + cache_path = _configured_catalog_cache_path() + try: + with urllib.request.urlopen(url, timeout=15) as response: + body = response.read().decode("utf-8") + if cache_path is not None: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(body, encoding="utf-8") + return body + except (OSError, urllib.error.URLError): + if cache_path is not None and cache_path.exists(): + return cache_path.read_text(encoding="utf-8") + raise + + +def _normalize_catalog_modules(payload: object) -> tuple[dict[str, object], ...]: + raw_items = payload.get("modules") if isinstance(payload, dict) else payload + if not isinstance(raw_items, list): + raise ValueError("Module package catalog must be a list or an object with a 'modules' list.") + return tuple(_normalize_catalog_item(item) for item in raw_items) + + +def _catalog_channel(payload: object) -> str | None: + if not isinstance(payload, dict): + return None + channel = payload.get("channel") + if channel is None: + return None + text = str(channel).strip() + return text or None + + +def _catalog_optional_text(payload: object, key: str) -> str | None: + if not isinstance(payload, dict): + return None + value = payload.get(key) + if value is None: + return None + text = str(value).strip() + return text or None + + +def _catalog_sequence(payload: object) -> int | None: + if not isinstance(payload, dict): + return None + value = payload.get("sequence") + if value is None: + return None + try: + sequence = int(value) + except (TypeError, ValueError): + raise ValueError("Catalog sequence must be an integer.") from None + if sequence < 0: + raise ValueError("Catalog sequence must not be negative.") + return sequence + + +def _catalog_signature_state(payload: object, *, trusted_keys: dict[str, str]) -> dict[str, object]: + if not isinstance(payload, dict): + return {"signed": False, "trusted": False, "key_id": None, "error": "List-style catalogs cannot be signed.", "fatal": False} + signatures = _catalog_signatures(payload) + if not signatures: + return {"signed": False, "trusted": False, "key_id": None, "error": "Catalog is unsigned.", "fatal": False} + errors: list[str] = [] + first_key_id: str | None = None + for signature in signatures: + result = _verify_catalog_signature(payload, signature, trusted_keys=trusted_keys) + key_id = result.get("key_id") + if first_key_id is None and isinstance(key_id, str): + first_key_id = key_id + if result.get("trusted"): + return {"signed": True, "trusted": True, "key_id": key_id, "error": None, "fatal": False} + if result.get("fatal"): + return {"signed": True, "trusted": False, "key_id": key_id, "error": result.get("error"), "fatal": True} + if result.get("error"): + errors.append(str(result["error"])) + return { + "signed": True, + "trusted": False, + "key_id": first_key_id, + "error": "; ".join(errors) if errors else "No catalog signature was trusted.", + "fatal": False, + } + + +def _catalog_signatures(payload: dict[str, object]) -> list[object]: + signatures: list[object] = [] + signature = payload.get("signature") + if signature is not None: + signatures.append(signature) + raw_signatures = payload.get("signatures") + if isinstance(raw_signatures, list): + signatures.extend(raw_signatures) + elif raw_signatures is not None: + signatures.append(raw_signatures) + return signatures + + +def _verify_catalog_signature(payload: dict[str, object], signature: object, *, trusted_keys: dict[str, str]) -> dict[str, object]: + if not isinstance(signature, dict): + return {"signed": True, "trusted": False, "key_id": None, "error": "Catalog signature must be an object.", "fatal": True} + algorithm = str(signature.get("algorithm") or "").strip().lower() + key_id = str(signature.get("key_id") or "").strip() + value = str(signature.get("value") or "").strip() + if algorithm != "ed25519": + return {"signed": True, "trusted": False, "key_id": key_id or None, "error": f"Unsupported catalog signature algorithm: {algorithm!r}", "fatal": True} + if not key_id or not value: + return {"signed": True, "trusted": False, "key_id": key_id or None, "error": "Catalog signature requires key_id and value.", "fatal": True} + trusted_key = trusted_keys.get(key_id) + if not trusted_key: + return {"signed": True, "trusted": False, "key_id": key_id, "error": f"Catalog signature key is not trusted: {key_id}", "fatal": False} + try: + public_key = Ed25519PublicKey.from_public_bytes(base64.b64decode(trusted_key, validate=True)) + signed_payload = dict(payload) + signed_payload.pop("signature", None) + signed_payload.pop("signatures", None) + public_key.verify(base64.b64decode(value, validate=True), _canonical_catalog_bytes(signed_payload)) + except (ValueError, binascii.Error, InvalidSignature) as exc: + return {"signed": True, "trusted": False, "key_id": key_id, "error": f"Catalog signature verification failed: {exc}", "fatal": True} + return {"signed": True, "trusted": True, "key_id": key_id, "error": None, "fatal": False} + + +def _catalog_freshness_state(payload: object) -> dict[str, object]: + if not isinstance(payload, dict): + return {"valid": True, "warnings": ()} + now = datetime.now(tz=UTC) + not_before = _parse_datetime(payload.get("not_before")) + generated_at = _parse_datetime(payload.get("generated_at")) + expires_at = _parse_datetime(payload.get("expires_at")) + if not_before is not None and now < not_before: + return {"valid": False, "error": f"Catalog is not valid before {not_before.isoformat()}."} + if expires_at is not None and now > expires_at: + return {"valid": False, "error": f"Catalog expired at {expires_at.isoformat()}."} + warnings: list[str] = [] + if generated_at is None: + warnings.append("Catalog has no generated_at timestamp.") + if expires_at is None: + warnings.append("Catalog has no expires_at timestamp; production catalogs should expire.") + return {"valid": True, "warnings": tuple(warnings)} + + +def _catalog_replay_state(*, channel: str | None, sequence: int | None) -> dict[str, object]: + state_path = _configured_sequence_state_path() + if state_path is None: + return {"valid": True, "warnings": ()} + if not channel: + return {"valid": False, "error": "Catalog sequence replay protection needs a channel."} + if sequence is None: + return {"valid": False, "error": "Catalog sequence replay protection needs a sequence."} + if not state_path.exists(): + return {"valid": True, "warnings": ()} + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {"valid": False, "error": f"Catalog sequence state file is not valid JSON: {state_path}"} + channels = state.get("channels") if isinstance(state, dict) else None + channel_state = channels.get(channel) if isinstance(channels, dict) else None + last_sequence = channel_state.get("last_sequence") if isinstance(channel_state, dict) else None + if last_sequence is None: + return {"valid": True, "warnings": ()} + try: + last = int(last_sequence) + except (TypeError, ValueError): + return {"valid": False, "error": f"Catalog sequence state for channel {channel!r} is invalid."} + if sequence < last: + return {"valid": False, "error": f"Catalog sequence {sequence} is older than accepted sequence {last} for channel {channel!r}."} + if sequence == last and _configured_enforce_sequence(): + return {"valid": False, "error": f"Catalog sequence {sequence} was already accepted for channel {channel!r}."} + return {"valid": True, "warnings": ()} + + +def _canonical_catalog_bytes(payload: object) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _load_private_key(path: Path) -> Ed25519PrivateKey: + private_key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(private_key, Ed25519PrivateKey): + raise ValueError("Catalog signing private key must be an Ed25519 PEM private key.") + return private_key + + +def _normalize_catalog_item(value: Any) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError("Module package catalog entries must be objects.") + module_id = _required_str(value, "module_id") + action = str(value.get("action") or "install") + if action not in {"install", "uninstall"}: + raise ValueError(f"Unsupported catalog action for {module_id!r}: {action!r}") + return { + "module_id": module_id, + "name": str(value.get("name") or module_id), + "description": _optional_str(value, "description"), + "version": _optional_str(value, "version"), + "action": action, + "python_package": _optional_str(value, "python_package"), + "python_ref": _optional_str(value, "python_ref"), + "webui_package": _optional_str(value, "webui_package"), + "webui_ref": _optional_str(value, "webui_ref"), + "license_features": _string_list(value.get("license_features")), + "notes": _optional_str(value, "notes"), + "tags": _string_list(value.get("tags")), + } + + +def _required_str(value: dict[str, Any], key: str) -> str: + item = _optional_str(value, key) + if not item: + raise ValueError(f"Module package catalog entry is missing {key!r}.") + return item + + +def _optional_str(value: dict[str, Any], key: str) -> str | None: + item = value.get(key) + if item is None: + return None + text = str(item).strip() + return text or None + + +def _string_list(value: Any) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("Module package catalog list fields must be lists.") + return [str(item).strip() for item in value if str(item).strip()] + + +def _parse_datetime(value: object) -> datetime | None: + if value is None: + return None + text = str(value).strip() + if not text: + return None + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError as exc: + raise ValueError(f"Invalid datetime value: {value!r}") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _catalog_source_exists(source: Path | str) -> bool: + if isinstance(source, str) and _is_http_url(source): + return True + return Path(source).exists() + + +def _catalog_source_type(source: Path | str | None) -> str | None: + if source is None: + return None + return "url" if isinstance(source, str) and _is_http_url(source) else "file" + + +def _is_http_url(value: str) -> bool: + return value.startswith(("https://", "http://")) + + +def _invalid_catalog_result( + source: Path | str | None, + *, + modules: tuple[dict[str, object], ...], + channel: str | None, + sequence: int | None, + generated_at: str | None, + expires_at: str | None, + signature_state: dict[str, object], + error: str, +) -> dict[str, object]: + return { + "valid": False, + "configured": source is not None, + "path": str(source) if source is not None else None, + "source": str(source) if source is not None else None, + "source_type": _catalog_source_type(source), + "modules": list(modules), + "channel": channel, + "sequence": sequence, + "generated_at": generated_at, + "expires_at": expires_at, + "signed": signature_state["signed"], + "trusted": signature_state["trusted"], + "key_id": signature_state["key_id"], + "warnings": [], + "error": error, + } diff --git a/src/govoplan_core/core/modules.py b/src/govoplan_core/core/modules.py index 76b7a92..c16324e 100644 --- a/src/govoplan_core/core/modules.py +++ b/src/govoplan_core/core/modules.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any, Literal, Protocol, TYPE_CHECKING @@ -63,15 +63,56 @@ class FrontendModule: module_id: str package_name: str | None = None asset_manifest: str | None = None + asset_manifest_signature: str | None = None + asset_manifest_public_key_id: str | None = None + asset_manifest_integrity: str | None = None + asset_manifest_contract_version: str = "1" routes: tuple[FrontendRoute, ...] = () nav_items: tuple[NavItem, ...] = () settings_routes: tuple[FrontendRoute, ...] = () + +@dataclass(frozen=True, slots=True) +class MigrationRetirementPlan: + supported: bool + summary: str + warnings: tuple[str, ...] = () + blocking_reason: str | None = None + destroy_data_supported: bool = False + destroy_data_summary: str | None = None + destroy_data_warnings: tuple[str, ...] = () + destroy_data_executor: MigrationRetirementExecutor | None = None + + +MigrationRetirementExecutor = Callable[[object, str], None] +MigrationRetirementProvider = Callable[[object | None, str], MigrationRetirementPlan] + + @dataclass(frozen=True, slots=True) class MigrationSpec: module_id: str metadata: object | None = None script_location: str | None = None + retirement_supported: bool = False + retirement_provider: MigrationRetirementProvider | None = None + retirement_notes: str | None = None + + +@dataclass(frozen=True, slots=True) +class ModuleCompatibility: + core_version_min: str | None = None + core_version_max: str | None = None + manifest_contract_version: str = "1" + + +@dataclass(frozen=True, slots=True) +class ModuleUninstallGuardResult: + severity: Literal["blocker", "warning", "info"] + code: str + message: str + + +UninstallGuardProvider = Callable[[object | None, str], Iterable[ModuleUninstallGuardResult]] @dataclass(frozen=True, slots=True) @@ -105,6 +146,7 @@ TenantSummaryProvider = Callable[[object, str], Mapping[str, int]] DeleteVetoProvider = Callable[[object, str, str], None] RouteFactory = Callable[[ModuleContext], "APIRouter"] CapabilityFactory = Callable[[ModuleContext], object] +LifecycleHook = Callable[[ModuleContext], None] @dataclass(frozen=True, slots=True) @@ -123,5 +165,8 @@ class ModuleManifest: resource_acl_providers: tuple[ResourceAclProvider, ...] = () tenant_summary_providers: tuple[TenantSummaryProvider, ...] = () delete_veto_providers: Mapping[str, Sequence[DeleteVetoProvider]] = field(default_factory=dict) + uninstall_guard_providers: tuple[UninstallGuardProvider, ...] = () capability_factories: Mapping[str, CapabilityFactory] = field(default_factory=dict) - + compatibility: ModuleCompatibility = field(default_factory=ModuleCompatibility) + on_activate: LifecycleHook | None = None + on_deactivate: LifecycleHook | None = None diff --git a/src/govoplan_core/core/registry.py b/src/govoplan_core/core/registry.py index 6679ed5..16e3857 100644 --- a/src/govoplan_core/core/registry.py +++ b/src/govoplan_core/core/registry.py @@ -55,6 +55,24 @@ class PlatformRegistry: self.register_capability_factory(manifest.id, name, factory) return manifest + def replace(self, manifests: Iterable[ModuleManifest]) -> RegistrySnapshot: + """Replace the active manifest set without replacing this registry object.""" + + replacement = PlatformRegistry() + for manifest in manifests: + replacement.register(manifest) + snapshot = replacement.validate() + + self._manifests = dict(replacement._manifests) + self._tenant_summary_providers = dict(replacement._tenant_summary_providers) + self._delete_veto_providers = defaultdict(list, { + resource_type: list(providers) + for resource_type, providers in replacement._delete_veto_providers.items() + }) + self._capability_factories = dict(replacement._capability_factories) + self._capabilities.clear() + return snapshot + def get(self, module_id: str) -> ModuleManifest | None: return self._manifests.get(module_id) @@ -189,4 +207,3 @@ class PlatformRegistry: unresolved = ", ".join(sorted(module_id for module_id, dependencies in incoming.items() if dependencies)) raise RegistryError(f"Module dependency cycle or unresolved dependency: {unresolved}") return (self._manifests[module_id] for module_id in ordered) - diff --git a/src/govoplan_core/core/runtime.py b/src/govoplan_core/core/runtime.py new file mode 100644 index 0000000..0c139b4 --- /dev/null +++ b/src/govoplan_core/core/runtime.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from govoplan_core.core.modules import ModuleContext + +_context: ModuleContext | None = None + + +def configure_runtime(context: ModuleContext) -> None: + global _context + _context = context + + +def get_runtime_context() -> ModuleContext | None: + return _context + + +def get_registry() -> object | None: + return _context.registry if _context is not None else None + diff --git a/src/govoplan_core/db/bootstrap.py b/src/govoplan_core/db/bootstrap.py index fa3c8e4..6359e4d 100644 --- a/src/govoplan_core/db/bootstrap.py +++ b/src/govoplan_core/db/bootstrap.py @@ -4,13 +4,16 @@ from dataclasses import dataclass from sqlalchemy.orm import Session -from govoplan_core.admin.service import ensure_default_roles, get_or_create_account -from govoplan_core.core.optional import reraise_unless_missing_package from govoplan_core.db.base import Base -from govoplan_core.db.models import Role, SystemRoleAssignment, Tenant, User, UserRoleAssignment from govoplan_core.db.session import get_database -from govoplan_core.security.api_keys import CreatedApiKey, create_api_key +from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids from govoplan_core.security.permissions import TENANT_SCOPES +from govoplan_core.server.registry import available_module_manifests, build_platform_registry +from govoplan_core.settings import settings +from govoplan_access.backend.admin.service import ensure_default_roles, get_or_create_account +from govoplan_access.backend.db.models import Role, SystemRoleAssignment, User, UserRoleAssignment +from govoplan_access.backend.security.api_keys import CreatedApiKey, create_api_key +from govoplan_tenancy.backend.db.models import Tenant DEFAULT_SCOPES = sorted(TENANT_SCOPES) @@ -22,29 +25,19 @@ class BootstrapResult: created_api_key: CreatedApiKey | None -def _import_optional_models(module_name: str) -> None: - try: - __import__(module_name) - except ModuleNotFoundError as exc: - reraise_unless_missing_package(exc, module_name.split(".", 1)[0]) - - def create_all_tables() -> None: - # Import models so SQLAlchemy sees all tables from installed modules. - from govoplan_core.db import models # noqa: F401 - from govoplan_core.access.db import models as access_models # noqa: F401 - from govoplan_core.access.db.base import AccessBase + # Build the configured registry so enabled module manifests register their + # model metadata with the shared SQLAlchemy base before create_all runs. + from govoplan_core.admin import models as core_admin_models # noqa: F401 - for module_name in ( - "govoplan_files.backend.db.models", - "govoplan_mail.backend.db.models", - "govoplan_campaign.backend.db.models", - ): - _import_optional_models(module_name) + raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules) + candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules) + available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True) + enabled_modules = load_startup_enabled_modules(settings.enabled_modules, available=available_modules) + build_platform_registry(enabled_modules) engine = get_database().engine Base.metadata.create_all(bind=engine) - AccessBase.metadata.create_all(bind=engine) def bootstrap_dev_data( @@ -73,7 +66,7 @@ def bootstrap_dev_data( ) # Keep development credentials deterministic when an old create_all database # is reused. This is guarded by the explicit dev bootstrap setting. - from govoplan_core.security.passwords import hash_password + from govoplan_access.backend.security.passwords import hash_password if not account.password_hash: account.password_hash = hash_password(user_password) diff --git a/src/govoplan_core/db/migrations.py b/src/govoplan_core/db/migrations.py index 9e02409..45dc6b0 100644 --- a/src/govoplan_core/db/migrations.py +++ b/src/govoplan_core/db/migrations.py @@ -10,8 +10,11 @@ from alembic.runtime.migration import MigrationContext from sqlalchemy import create_engine, inspect, text from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan +from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids +from govoplan_core.db.session import configure_database from govoplan_core.server.default_config import get_server_config -from govoplan_core.server.registry import build_platform_registry +from govoplan_core.server.config import ManifestFactory +from govoplan_core.server.registry import available_module_manifests, build_platform_registry from govoplan_core.settings import settings # Historic development databases could be created partly through Alembic and @@ -133,11 +136,32 @@ class MigrationResult: current_revision: str | None -def registered_module_migration_plan() -> MigrationMetadataPlan: +def registered_module_migration_plan( + database_url: str | None = None, + *, + enabled_modules: tuple[str, ...] | list[str] | None = None, + manifest_factories: tuple[ManifestFactory, ...] = (), +) -> MigrationMetadataPlan: server_config = get_server_config() + active_database_url = database_url or settings.database_url + if active_database_url: + configure_database(active_database_url) + active_manifest_factories = manifest_factories or tuple(server_config.manifest_factories) + raw_enabled_modules = tuple(enabled_modules) if enabled_modules is not None else load_startup_enabled_modules(server_config.enabled_modules) + candidate_modules = startup_candidate_module_ids(server_config.enabled_modules, raw_enabled_modules) + available_modules = available_module_manifests( + active_manifest_factories, + enabled_modules=candidate_modules, + ignore_load_errors=True, + ) + active_enabled_modules = ( + tuple(enabled_modules) + if enabled_modules is not None + else load_startup_enabled_modules(server_config.enabled_modules, available=available_modules) + ) registry = build_platform_registry( - server_config.enabled_modules, - manifest_factories=server_config.manifest_factories, + active_enabled_modules, + manifest_factories=active_manifest_factories, ) return migration_metadata_plan(registry) @@ -146,18 +170,32 @@ def _repo_root() -> Path: return Path(__file__).resolve().parents[3] -def alembic_config(*, database_url: str | None = None) -> Config: +def alembic_config( + *, + database_url: str | None = None, + enabled_modules: tuple[str, ...] | list[str] | None = None, + manifest_factories: tuple[ManifestFactory, ...] = (), +) -> Config: root = _repo_root() config = Config(str(root / "alembic.ini")) config.set_main_option("script_location", str(root / "alembic")) - plan = registered_module_migration_plan() + active_database_url = database_url or settings.database_url + plan = registered_module_migration_plan( + active_database_url, + enabled_modules=enabled_modules, + manifest_factories=manifest_factories, + ) version_locations = [str(root / "alembic" / "versions")] version_locations.extend(location for location in plan.script_locations if location) config.set_main_option("version_locations", os.pathsep.join(dict.fromkeys(version_locations))) config.set_main_option("version_path_separator", "os") - config.attributes["database_url"] = database_url or settings.database_url + config.attributes["database_url"] = active_database_url + if enabled_modules is not None: + config.attributes["enabled_modules"] = tuple(enabled_modules) + if manifest_factories: + config.attributes["manifest_factories"] = tuple(manifest_factories) return config @@ -166,7 +204,12 @@ def database_revision(database_url: str | None = None) -> str | None: engine = create_engine(url) try: with engine.connect() as connection: - return MigrationContext.configure(connection).get_current_revision() + heads = MigrationContext.configure(connection).get_current_heads() + if not heads: + return None + if len(heads) == 1: + return heads[0] + return ",".join(sorted(heads)) finally: engine.dispose() @@ -227,7 +270,9 @@ def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | engine = create_engine(url) try: with engine.connect() as connection: - current = MigrationContext.configure(connection).get_current_revision() + heads = MigrationContext.configure(connection).get_current_heads() + current = heads[0] if len(heads) == 1 else None + has_no_revision = len(heads) == 0 schema = inspect(connection) tables = set(schema.get_table_names()) @@ -259,10 +304,10 @@ def reconcile_legacy_create_all_schema(database_url: str | None = None) -> str | # run normally. _backfill_user_lock_state_for_create_all_schema(url) target = REVISION_HIERARCHICAL_SETTINGS - elif current is None and has_create_all_hierarchical_schema: + elif has_no_revision and has_create_all_hierarchical_schema: _backfill_user_lock_state_for_create_all_schema(url) target = REVISION_HIERARCHICAL_SETTINGS - elif current is None and has_file_storage and has_file_folders: + elif has_no_revision and has_file_storage and has_file_folders: # This is the other create_all-only development shape. The strict # column checks above ensure that we only stamp a complete known schema. target = REVISION_FILE_FOLDERS @@ -278,11 +323,20 @@ def migrate_database( *, database_url: str | None = None, reconcile_legacy_schema: bool = True, + enabled_modules: tuple[str, ...] | list[str] | None = None, + manifest_factories: tuple[ManifestFactory, ...] = (), ) -> MigrationResult: url = database_url or settings.database_url previous = database_revision(url) reconciled = reconcile_legacy_create_all_schema(url) if reconcile_legacy_schema else None - command.upgrade(alembic_config(database_url=url), "head") + command.upgrade( + alembic_config( + database_url=url, + enabled_modules=enabled_modules, + manifest_factories=manifest_factories, + ), + "heads", + ) current = database_revision(url) return MigrationResult( previous_revision=previous, diff --git a/src/govoplan_core/db/models.py b/src/govoplan_core/db/models.py deleted file mode 100644 index cca07a1..0000000 --- a/src/govoplan_core/db/models.py +++ /dev/null @@ -1,271 +0,0 @@ -from __future__ import annotations - -import uuid -from datetime import datetime -from typing import Any - -from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint, JSON, text -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from govoplan_core.db.base import Base, TimestampMixin - - -def new_uuid() -> str: - return str(uuid.uuid4()) - - -class Account(Base, TimestampMixin): - """Global login identity shared by one or more tenant memberships.""" - - __tablename__ = "accounts" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - email: Mapped[str] = mapped_column(String(320), nullable=False) - normalized_email: Mapped[str] = mapped_column(String(320), unique=True, nullable=False, index=True) - display_name: Mapped[str | None] = mapped_column(String(255)) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - auth_provider: Mapped[str] = mapped_column(String(50), default="local", nullable=False) - password_hash: Mapped[str | None] = mapped_column(String(500)) - password_reset_required: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - - memberships: Mapped[list[User]] = relationship(back_populates="account") - auth_sessions: Mapped[list[AuthSession]] = relationship(back_populates="account", cascade="all, delete-orphan") - system_role_assignments: Mapped[list[SystemRoleAssignment]] = relationship( - back_populates="account", cascade="all, delete-orphan" - ) - - -class Tenant(Base, TimestampMixin): - __tablename__ = "tenants" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - slug: Mapped[str] = mapped_column(String(100), unique=True, nullable=False, index=True) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - allow_custom_groups: Mapped[bool | None] = mapped_column(Boolean, nullable=True) - allow_custom_roles: Mapped[bool | None] = mapped_column(Boolean, nullable=True) - allow_api_keys: Mapped[bool | None] = mapped_column(Boolean, nullable=True) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - - users: Mapped[list[User]] = relationship(back_populates="tenant", cascade="all, delete-orphan") - - -class User(Base, TimestampMixin): - __tablename__ = "users" - __table_args__ = ( - UniqueConstraint("tenant_id", "email", name="uq_users_tenant_email"), - UniqueConstraint("tenant_id", "account_id", name="uq_users_tenant_account"), - ) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - account_id: Mapped[str] = mapped_column(ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True) - email: Mapped[str] = mapped_column(String(320), nullable=False, index=True) - display_name: Mapped[str | None] = mapped_column(String(255)) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - is_tenant_admin: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - auth_provider: Mapped[str] = mapped_column(String(50), default="local", nullable=False) - password_hash: Mapped[str | None] = mapped_column(String(500)) - last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - mail_profile_policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - tenant: Mapped[Tenant] = relationship(back_populates="users") - account: Mapped[Account] = relationship(back_populates="memberships") - api_keys: Mapped[list[ApiKey]] = relationship(back_populates="user", cascade="all, delete-orphan") - auth_sessions: Mapped[list[AuthSession]] = relationship(back_populates="user", cascade="all, delete-orphan") - - -class Group(Base, TimestampMixin): - __tablename__ = "groups" - __table_args__ = (UniqueConstraint("tenant_id", "slug", name="uq_groups_tenant_slug"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - slug: Mapped[str] = mapped_column(String(100), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - system_template_id: Mapped[str | None] = mapped_column(ForeignKey("governance_templates.id", ondelete="SET NULL"), nullable=True, index=True) - system_required: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - mail_profile_policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - -class Role(Base, TimestampMixin): - __tablename__ = "roles" - __table_args__ = ( - UniqueConstraint("tenant_id", "slug", name="uq_roles_tenant_slug"), - Index( - "uq_roles_system_slug", - "slug", - unique=True, - sqlite_where=text("tenant_id IS NULL"), - postgresql_where=text("tenant_id IS NULL"), - ), - ) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=True, index=True) - slug: Mapped[str] = mapped_column(String(100), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - permissions: Mapped[list[str]] = mapped_column(JSON, default=list) - is_builtin: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - is_assignable: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - system_template_id: Mapped[str | None] = mapped_column(ForeignKey("governance_templates.id", ondelete="SET NULL"), nullable=True, index=True) - system_required: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - - -class SystemSettings(Base, TimestampMixin): - __tablename__ = "system_settings" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default="global") - default_locale: Mapped[str] = mapped_column(String(20), default="en", nullable=False) - allow_tenant_custom_groups: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - allow_tenant_custom_roles: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - allow_tenant_api_keys: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - settings: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) - - -class GovernanceTemplate(Base, TimestampMixin): - __tablename__ = "governance_templates" - __table_args__ = (UniqueConstraint("kind", "slug", name="uq_governance_templates_kind_slug"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - kind: Mapped[str] = mapped_column(String(20), nullable=False, index=True) - slug: Mapped[str] = mapped_column(String(100), nullable=False) - name: Mapped[str] = mapped_column(String(255), nullable=False) - description: Mapped[str | None] = mapped_column(Text) - permissions: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) - is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) - - -class GovernanceTemplateAssignment(Base, TimestampMixin): - __tablename__ = "governance_template_assignments" - __table_args__ = (UniqueConstraint("template_id", "tenant_id", name="uq_governance_template_tenant"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - template_id: Mapped[str] = mapped_column(ForeignKey("governance_templates.id", ondelete="CASCADE"), nullable=False, index=True) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - mode: Mapped[str] = mapped_column(String(20), default="available", nullable=False) - - -class SystemRoleAssignment(Base, TimestampMixin): - __tablename__ = "system_role_assignments" - __table_args__ = (UniqueConstraint("account_id", "role_id", name="uq_system_role_assignments"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - account_id: Mapped[str] = mapped_column(ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True) - role_id: Mapped[str] = mapped_column(ForeignKey("roles.id", ondelete="CASCADE"), nullable=False, index=True) - - account: Mapped[Account] = relationship(back_populates="system_role_assignments") - role: Mapped[Role] = relationship() - - -class UserGroupMembership(Base, TimestampMixin): - __tablename__ = "user_group_memberships" - __table_args__ = (UniqueConstraint("tenant_id", "user_id", "group_id", name="uq_user_group_memberships"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - user_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - group_id: Mapped[str] = mapped_column(ForeignKey("groups.id", ondelete="CASCADE"), nullable=False, index=True) - - -class UserRoleAssignment(Base, TimestampMixin): - __tablename__ = "user_role_assignments" - __table_args__ = (UniqueConstraint("tenant_id", "user_id", "role_id", name="uq_user_role_assignments"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - user_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - role_id: Mapped[str] = mapped_column(ForeignKey("roles.id", ondelete="CASCADE"), nullable=False, index=True) - - -class GroupRoleAssignment(Base, TimestampMixin): - __tablename__ = "group_role_assignments" - __table_args__ = (UniqueConstraint("tenant_id", "group_id", "role_id", name="uq_group_role_assignments"),) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - group_id: Mapped[str] = mapped_column(ForeignKey("groups.id", ondelete="CASCADE"), nullable=False, index=True) - role_id: Mapped[str] = mapped_column(ForeignKey("roles.id", ondelete="CASCADE"), nullable=False, index=True) - - -class ApiKey(Base, TimestampMixin): - __tablename__ = "api_keys" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - user_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - name: Mapped[str] = mapped_column(String(255), nullable=False) - prefix: Mapped[str] = mapped_column(String(16), nullable=False, index=True) - key_hash: Mapped[str] = mapped_column(String(128), nullable=False) - scopes: Mapped[list[str]] = mapped_column(JSON, default=list) - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - - user: Mapped[User] = relationship(back_populates="api_keys") - - - - -class AuthSession(Base, TimestampMixin): - __tablename__ = "auth_sessions" - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - tenant_id: Mapped[str] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True) - user_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - account_id: Mapped[str] = mapped_column(ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash: Mapped[str] = mapped_column(String(128), nullable=False, unique=True, index=True) - csrf_token_hash: Mapped[str | None] = mapped_column(String(128), nullable=True) - expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True) - last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) - user_agent: Mapped[str | None] = mapped_column(String(500)) - ip_address: Mapped[str | None] = mapped_column(String(100)) - - user: Mapped[User] = relationship(back_populates="auth_sessions") - account: Mapped[Account] = relationship(back_populates="auth_sessions") - - -class AuditLog(Base, TimestampMixin): - __tablename__ = "audit_log" - __table_args__ = ( - Index("ix_audit_log_scope_created_at", "scope", "created_at"), - Index("ix_audit_log_tenant_scope_created_at", "tenant_id", "scope", "created_at"), - ) - - id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) - scope: Mapped[str] = mapped_column(String(20), default="tenant", nullable=False, index=True) - tenant_id: Mapped[str | None] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=True, index=True) - user_id: Mapped[str | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) - api_key_id: Mapped[str | None] = mapped_column(ForeignKey("api_keys.id", ondelete="SET NULL"), nullable=True, index=True) - action: Mapped[str] = mapped_column(String(100), nullable=False, index=True) - object_type: Mapped[str | None] = mapped_column(String(100), index=True) - object_id: Mapped[str | None] = mapped_column(String(100), index=True) - details: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) - - -__all__ = [ - "Account", - "ApiKey", - "AuditLog", - "AuthSession", - "GovernanceTemplate", - "GovernanceTemplateAssignment", - "Group", - "GroupRoleAssignment", - "Role", - "SystemRoleAssignment", - "SystemSettings", - "Tenant", - "User", - "UserGroupMembership", - "UserRoleAssignment", -] diff --git a/src/govoplan_core/devserver.py b/src/govoplan_core/devserver.py index ca80911..ba8314d 100644 --- a/src/govoplan_core/devserver.py +++ b/src/govoplan_core/devserver.py @@ -11,10 +11,12 @@ from pathlib import Path from typing import Any from govoplan_core.core.discovery import iter_module_entry_points +from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids from govoplan_core.core.modules import ModuleManifest from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.session import configure_database from govoplan_core.server.config import GovoplanServerConfig, import_object, load_server_config -from govoplan_core.server.registry import build_platform_registry +from govoplan_core.server.registry import available_module_manifests, build_platform_registry DEFAULT_APP = "govoplan_core.server.app:app" DEFAULT_CONFIG = "govoplan_core.server.default_config:get_server_config" @@ -251,10 +253,20 @@ def prepare_devserver(config_path: str | None, *, extra_reload_dirs: Sequence[st config = load_server_config(config_path) database_url = str(getattr(config.settings, "database_url", database_url) or "") validate_sqlite_database_url(database_url) + if database_url: + configure_database(database_url) if bootstrap_db_path is None: bootstrap_db_path = enable_dev_bootstrap_for_missing_sqlite(database_url) apply_detected_dev_bootstrap(config, bootstrap_db_path) - registry = build_platform_registry(config.enabled_modules, manifest_factories=config.manifest_factories) + raw_enabled_modules = load_startup_enabled_modules(config.enabled_modules) + candidate_modules = startup_candidate_module_ids(config.enabled_modules, raw_enabled_modules) + available_modules = available_module_manifests( + config.manifest_factories, + enabled_modules=candidate_modules, + ignore_load_errors=True, + ) + enabled_modules = load_startup_enabled_modules(config.enabled_modules, available=available_modules) + registry = build_platform_registry(enabled_modules, manifest_factories=config.manifest_factories) reload_dirs = build_reload_dirs(config, config_path=config_path, registry=registry, extra_dirs=extra_reload_dirs) return DevserverState( config_path=config_path, diff --git a/src/govoplan_core/privacy/retention.py b/src/govoplan_core/privacy/retention.py index 479aeb4..d906ed0 100644 --- a/src/govoplan_core/privacy/retention.py +++ b/src/govoplan_core/privacy/retention.py @@ -1,6 +1,5 @@ from __future__ import annotations -import copy import hashlib import json from datetime import datetime, timedelta, timezone @@ -10,10 +9,20 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy.orm import Session -from govoplan_core.admin.governance import get_system_settings -from govoplan_core.core.optional import reraise_unless_missing_package -from govoplan_core.db.models import AuditLog, Group, SystemSettings, Tenant, User +from govoplan_core.admin.settings import get_system_settings +from govoplan_core.core.campaigns import ( + CAPABILITY_CAMPAIGNS_POLICY_CONTEXT, + CAPABILITY_CAMPAIGNS_RETENTION, + CampaignPolicyContext, + CampaignPolicyContextProvider, + CampaignRetentionProvider, +) +from govoplan_core.core.runtime import get_registry +from govoplan_access.backend.db.models import Group, User +from govoplan_audit.backend.db.models import AuditLog +from govoplan_core.admin.models import SystemSettings from govoplan_core.settings import settings +from govoplan_tenancy.backend.db.models import Tenant PRIVACY_POLICY_SETTINGS_KEY = "privacy_retention_policy" RETENTION_DAY_KEYS = ( @@ -48,21 +57,6 @@ def _normalize_allow_lower_level_limits(value: Any, *, fill_defaults: bool) -> d normalized[clean_key] = bool(allowed) return normalized -FINAL_VERSION_STATES = { - "completed", - "partially_completed", - "outcome_unknown", - "failed", - "cancelled", - "archived", -} -FINAL_EML_SEND_STATUSES = { - "smtp_accepted", - "sent", - "failed_permanent", - "cancelled", - "outcome_unknown", -} AUDIT_DETAIL_REDACTION_KEYS = { "attachments", "bcc", @@ -148,20 +142,34 @@ class PrivacyPolicyError(RuntimeError): pass -def _campaign_models(): - try: - from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus - except ModuleNotFoundError as exc: - reraise_unless_missing_package(exc, "govoplan_campaign") - raise PrivacyPolicyError("Campaign module is not installed") from exc - return Campaign, CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus - - -def _campaign_models_or_none(): - try: - return _campaign_models() - except PrivacyPolicyError: +def _campaign_policy_provider() -> CampaignPolicyContextProvider | None: + registry = get_registry() + if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT): return None + capability = registry.require_capability(CAPABILITY_CAMPAIGNS_POLICY_CONTEXT) + if not isinstance(capability, CampaignPolicyContextProvider): + raise PrivacyPolicyError("Campaign policy context capability is invalid") + return capability + + +def _campaign_retention_provider() -> CampaignRetentionProvider | None: + registry = get_registry() + if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(CAPABILITY_CAMPAIGNS_RETENTION): + return None + capability = registry.require_capability(CAPABILITY_CAMPAIGNS_RETENTION) + if not isinstance(capability, CampaignRetentionProvider): + raise PrivacyPolicyError("Campaign retention capability is invalid") + return capability + + +def _campaign_policy_context(session: Session, *, campaign_id: str, tenant_id: str | None = None) -> CampaignPolicyContext: + provider = _campaign_policy_provider() + if provider is None: + raise PrivacyPolicyError("Campaign module is not installed") + context = provider.get_campaign_policy_context(session, tenant_id=tenant_id, campaign_id=campaign_id) + if context is None: + raise PrivacyPolicyError("Campaign not found for privacy policy") + return context def _utcnow() -> datetime: @@ -267,12 +275,9 @@ def effective_privacy_policy( campaign_id: str | None = None, ) -> PrivacyRetentionPolicy: policy = privacy_policy_from_session(session) - campaign: Any | None = None + campaign: CampaignPolicyContext | None = None if campaign_id: - Campaign, _, _, _, _ = _campaign_models() - campaign = session.get(Campaign, campaign_id) - if campaign is None: - raise PrivacyPolicyError("Campaign not found for privacy policy") + campaign = _campaign_policy_context(session, campaign_id=campaign_id) tenant_id = campaign.tenant_id owner_user_id = campaign.owner_user_id owner_group_id = campaign.owner_group_id @@ -290,7 +295,7 @@ def effective_privacy_policy( if group and (not tenant_id or group.tenant_id == tenant_id): policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(group.settings or {})) if campaign is not None: - policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(campaign.settings or {})) + policy = _merge_privacy_policy(policy, _privacy_policy_patch_from_settings(dict(campaign.settings or {}))) return policy @@ -307,10 +312,7 @@ def parent_privacy_policy(session: Session, *, tenant_id: str, scope_type: str, return policy if clean_scope != "campaign" or not scope_id: return policy - Campaign, _, _, _, _ = _campaign_models() - campaign = session.get(Campaign, scope_id) - if campaign is None or campaign.tenant_id != tenant_id: - raise PrivacyPolicyError("Campaign not found for privacy policy") + campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) if campaign.owner_user_id: user = session.get(User, campaign.owner_user_id) if user and user.tenant_id == tenant_id: @@ -356,12 +358,9 @@ def effective_privacy_policy_sources( ) -> list[dict[str, Any]]: system_settings = get_system_settings(session) sources = [_retention_policy_source_step("system", "System", None, _privacy_policy_patch_from_settings(system_settings.settings or {}), baseline=True)] - campaign: Any | None = None + campaign: CampaignPolicyContext | None = None if campaign_id: - Campaign, _, _, _, _ = _campaign_models() - campaign = session.get(Campaign, campaign_id) - if campaign is None: - raise PrivacyPolicyError("Campaign not found for privacy policy") + campaign = _campaign_policy_context(session, campaign_id=campaign_id) tenant_id = campaign.tenant_id owner_user_id = campaign.owner_user_id owner_group_id = campaign.owner_group_id @@ -379,7 +378,7 @@ def effective_privacy_policy_sources( if group and (not tenant_id or group.tenant_id == tenant_id): sources.append(_retention_policy_source_step("group", "Owner group", group.id, _privacy_policy_patch_from_settings(group.settings or {}))) if campaign is not None: - sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(campaign.settings or {}))) + sources.append(_retention_policy_source_step("campaign", "Campaign", campaign.id, _privacy_policy_patch_from_settings(dict(campaign.settings or {})))) return sources @@ -399,10 +398,7 @@ def parent_privacy_policy_sources(session: Session, *, tenant_id: str, scope_typ return sources if clean_scope != "campaign" or not scope_id: return sources - Campaign, _, _, _, _ = _campaign_models() - campaign = session.get(Campaign, scope_id) - if campaign is None or campaign.tenant_id != tenant_id: - raise PrivacyPolicyError("Campaign not found for privacy policy") + campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) if campaign.owner_user_id: user = session.get(User, campaign.owner_user_id) if user and user.tenant_id == tenant_id: @@ -435,11 +431,8 @@ def get_privacy_policy_for_scope(session: Session, *, tenant_id: str, scope_type raise PrivacyPolicyError("Group privacy policy not found") return _privacy_policy_patch_from_settings(group.settings or {}) if clean_scope == "campaign": - Campaign, _, _, _, _ = _campaign_models() - campaign = session.get(Campaign, scope_id) - if campaign is None or campaign.tenant_id != tenant_id: - raise PrivacyPolicyError("Campaign privacy policy not found") - return _privacy_policy_patch_from_settings(campaign.settings or {}) + campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) + return _privacy_policy_patch_from_settings(dict(campaign.settings or {})) raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign") @@ -483,12 +476,15 @@ def set_privacy_policy_for_scope( session.add(group) return patch if clean_scope == "campaign": - Campaign, _, _, _, _ = _campaign_models() - campaign = session.get(Campaign, scope_id) - if campaign is None or campaign.tenant_id != tenant_id: - raise PrivacyPolicyError("Campaign privacy policy not found") - campaign.settings = _set_settings_privacy_policy(campaign.settings, patch) - session.add(campaign) + provider = _campaign_policy_provider() + if provider is None: + raise PrivacyPolicyError("Campaign module is not installed") + campaign = _campaign_policy_context(session, tenant_id=tenant_id, campaign_id=scope_id) + settings_payload = _set_settings_privacy_policy(dict(campaign.settings or {}), patch) + try: + provider.set_campaign_settings(session, tenant_id=tenant_id, campaign_id=scope_id, settings=settings_payload) + except ValueError as exc: + raise PrivacyPolicyError("Campaign privacy policy not found") from exc return patch raise PrivacyPolicyError("Privacy policy scope must be system, tenant, user, group or campaign") @@ -540,10 +536,29 @@ def apply_retention_policy(session: Session, *, dry_run: bool = True) -> dict[st now = _utcnow() policy = privacy_policy_from_session(session) cutoffs = _system_cutoffs(policy, now=now) + policy_cache: dict[str, PrivacyRetentionPolicy] = {} + campaign_retention = _campaign_retention_provider() + campaign_counts = { + "raw_campaign_json": {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}, + "generated_eml": {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}, + "stored_report_detail": {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}, + } + if campaign_retention is not None: + campaign_counts.update( + { + key: dict(value) + for key, value in campaign_retention.apply_retention( + session, + dry_run=dry_run, + now=now, + policy_for_campaign_id=lambda campaign_id: _campaign_policy_for_id(session, campaign_id, policy_cache), + ).items() + } + ) counts = { - "raw_campaign_json": _apply_raw_json_retention(session, dry_run=dry_run, now=now), - "generated_eml": _apply_eml_retention(session, dry_run=dry_run, now=now), - "stored_report_detail": _apply_report_detail_retention(session, dry_run=dry_run, now=now), + "raw_campaign_json": campaign_counts["raw_campaign_json"], + "generated_eml": campaign_counts["generated_eml"], + "stored_report_detail": campaign_counts["stored_report_detail"], "mock_mailbox": _apply_mock_mailbox_retention(cutoffs["mock_mailbox"], dry_run=dry_run), "audit_detail": _apply_audit_detail_retention(session, dry_run=dry_run, now=now), } @@ -564,142 +579,6 @@ def _campaign_policy_for_id(session: Session, campaign_id: str | None, cache: di return cache[campaign_id] -def _apply_raw_json_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: - result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0} - models = _campaign_models_or_none() - if models is None: - return result - _, _, CampaignVersion, _, _ = models - policy_cache: dict[str, PrivacyRetentionPolicy] = {} - versions = ( - session.query(CampaignVersion) - .order_by(CampaignVersion.updated_at.asc()) - .all() - ) - for version in versions: - policy = _campaign_policy_for_id(session, version.campaign_id, policy_cache) - cutoff = _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now) - if not _is_before_cutoff(version.updated_at, cutoff): - continue - raw_json = version.raw_json if isinstance(version.raw_json, dict) else {} - if raw_json.get("_retention", {}).get("raw_json_redacted"): - result["already_redacted"] += 1 - continue - if version.workflow_state not in FINAL_VERSION_STATES: - result["skipped_not_final"] += 1 - continue - result["eligible"] += 1 - if dry_run: - continue - snapshot = version.execution_snapshot if isinstance(version.execution_snapshot, dict) else {} - version.raw_json = { - "version": version.schema_version or "1.0", - "_retention": { - "raw_json_redacted": True, - "redacted_at": now.isoformat(), - "original_sha256": snapshot.get("campaign_json_sha256") or _json_sha256(raw_json), - }, - } - session.add(version) - result["redacted"] += 1 - return result - - -def _apply_eml_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: - result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0} - models = _campaign_models_or_none() - if models is None: - return result - _, CampaignJob, _, JobImapStatus, JobQueueStatus = models - policy_cache: dict[str, PrivacyRetentionPolicy] = {} - jobs = ( - session.query(CampaignJob) - .filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None))) - .order_by(CampaignJob.updated_at.asc()) - .all() - ) - for job in jobs: - policy = _campaign_policy_for_id(session, job.campaign_id, policy_cache) - cutoff = _cutoff(policy.generated_eml_retention_days, now=now) - if not _is_before_cutoff(job.updated_at, cutoff): - continue - if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES: - result["skipped_not_final"] += 1 - continue - if job.imap_status == JobImapStatus.PENDING.value: - result["skipped_not_final"] += 1 - continue - result["eligible"] += 1 - if dry_run: - continue - if job.eml_local_path: - path = Path(job.eml_local_path) - if path.exists(): - path.unlink() - result["files_deleted"] += 1 - else: - result["files_missing"] += 1 - job.eml_local_path = None - job.eml_storage_key = None - session.add(job) - result["metadata_cleared"] += 1 - return result - - -def _apply_report_detail_retention(session: Session, *, dry_run: bool, now: datetime) -> dict[str, int]: - result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0} - models = _campaign_models_or_none() - if models is None: - return result - _, _, CampaignVersion, _, _ = models - policy_cache: dict[str, PrivacyRetentionPolicy] = {} - versions = ( - session.query(CampaignVersion) - .order_by(CampaignVersion.updated_at.asc()) - .all() - ) - for version in versions: - policy = _campaign_policy_for_id(session, version.campaign_id, policy_cache) - cutoff = _cutoff(policy.stored_report_detail_retention_days, now=now) - if not _is_before_cutoff(version.updated_at, cutoff): - continue - next_validation, validation_changed = _redacted_report_summary(version.validation_summary, now=now) - next_build, build_changed = _redacted_report_summary(version.build_summary, now=now) - if not validation_changed and not build_changed: - if _summary_was_redacted(version.validation_summary) or _summary_was_redacted(version.build_summary): - result["already_redacted"] += 1 - continue - result["eligible_versions"] += 1 - if dry_run: - continue - version.validation_summary = next_validation - version.build_summary = next_build - session.add(version) - result["summaries_redacted"] += int(validation_changed) + int(build_changed) - return result - - -def _summary_was_redacted(summary: Any) -> bool: - return isinstance(summary, dict) and bool(summary.get("_retention", {}).get("report_detail_redacted")) - - -def _redacted_report_summary(summary: Any, *, now: datetime) -> tuple[dict[str, Any] | None, bool]: - if not isinstance(summary, dict): - return summary, False - if _summary_was_redacted(summary): - return summary, False - detail_keys = {"messages", "issues", "recent_failures", "jobs"} - if not any(key in summary for key in detail_keys): - return summary, False - next_summary = copy.deepcopy(summary) - for key in detail_keys: - next_summary.pop(key, None) - retention = dict(next_summary.get("_retention") or {}) - retention.update({"report_detail_redacted": True, "redacted_at": now.isoformat()}) - next_summary["_retention"] = retention - return next_summary, True - - def _apply_mock_mailbox_retention(cutoff: datetime | None, *, dry_run: bool) -> dict[str, int]: result = {"eligible_records": 0, "json_deleted": 0, "eml_deleted": 0} if cutoff is None: @@ -744,12 +623,11 @@ def _parse_datetime(value: Any) -> datetime | None: def _privacy_policy_for_audit_item(session: Session, item: AuditLog, campaign_cache: dict[str, PrivacyRetentionPolicy], tenant_cache: dict[str, PrivacyRetentionPolicy]) -> PrivacyRetentionPolicy: if item.object_type == "campaign" and item.object_id: - models = _campaign_models_or_none() - if models is not None: - Campaign, _, _, _, _ = models - campaign = session.get(Campaign, str(item.object_id)) - if campaign is not None: - return _campaign_policy_for_id(session, campaign.id, campaign_cache) + provider = _campaign_policy_provider() + if provider is not None: + context = provider.get_campaign_policy_context(session, campaign_id=str(item.object_id)) + if context is not None: + return _campaign_policy_for_id(session, context.id, campaign_cache) if item.tenant_id: if item.tenant_id not in tenant_cache: tenant_cache[item.tenant_id] = effective_privacy_policy(session, tenant_id=item.tenant_id) diff --git a/src/govoplan_core/security/api_keys.py b/src/govoplan_core/security/api_keys.py deleted file mode 100644 index b274d80..0000000 --- a/src/govoplan_core/security/api_keys.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -from sqlalchemy.orm import Session - -from govoplan_core.db.models import ApiKey, User -from govoplan_core.access.auth.tokens import generate_secret, hash_secret, verify_secret -from govoplan_core.security.time import ensure_aware_utc, utc_now - -API_KEY_PREFIX_LENGTH = 12 -API_KEY_RANDOM_BYTES = 32 - - -@dataclass(slots=True) -class CreatedApiKey: - model: ApiKey - secret: str - - -def hash_api_key(secret: str) -> str: - return hash_secret(secret) - - -def verify_api_key(secret: str, expected_hash: str) -> bool: - return verify_secret(secret, expected_hash) - - -def generate_api_key_secret() -> str: - return generate_secret("mm_", random_bytes=API_KEY_RANDOM_BYTES) - - -def api_key_prefix(secret: str) -> str: - # Prefix is only a lookup helper and must not be enough to authenticate. - return secret[:API_KEY_PREFIX_LENGTH] - - -def create_api_key( - session: Session, - *, - user: User, - name: str, - scopes: list[str], - secret: str | None = None, - expires_at: datetime | None = None, -) -> CreatedApiKey: - secret = secret or generate_api_key_secret() - model = ApiKey( - tenant_id=user.tenant_id, - user_id=user.id, - name=name, - prefix=api_key_prefix(secret), - key_hash=hash_api_key(secret), - scopes=scopes, - expires_at=expires_at, - ) - session.add(model) - session.flush() - return CreatedApiKey(model=model, secret=secret) - - -def authenticate_api_key(session: Session, secret: str) -> ApiKey | None: - prefix = api_key_prefix(secret) - candidates = session.query(ApiKey).filter(ApiKey.prefix == prefix, ApiKey.revoked_at.is_(None)).all() - now = utc_now() - for candidate in candidates: - expires_at = ensure_aware_utc(candidate.expires_at) - if expires_at and expires_at < now: - continue - if verify_api_key(secret, candidate.key_hash): - candidate.last_used_at = now - session.add(candidate) - return candidate - return None - - -def has_scope(api_key: ApiKey, required_scope: str) -> bool: - scopes = set(api_key.scopes or []) - return "*" in scopes or required_scope in scopes diff --git a/src/govoplan_core/security/passwords.py b/src/govoplan_core/security/passwords.py deleted file mode 100644 index da2ca73..0000000 --- a/src/govoplan_core/security/passwords.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import base64 -import hashlib -import hmac -import os - -_ALGORITHM = "pbkdf2_sha256" -_DEFAULT_ITERATIONS = 260_000 -_SALT_BYTES = 16 - - -def hash_password(password: str, *, iterations: int = _DEFAULT_ITERATIONS) -> str: - salt = os.urandom(_SALT_BYTES) - digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) - return "$".join([ - _ALGORITHM, - str(iterations), - base64.b64encode(salt).decode("ascii"), - base64.b64encode(digest).decode("ascii"), - ]) - - -def verify_password(password: str, encoded: str | None) -> bool: - if not encoded: - return False - try: - algorithm, iterations_text, salt_b64, digest_b64 = encoded.split("$", 3) - if algorithm != _ALGORITHM: - return False - iterations = int(iterations_text) - salt = base64.b64decode(salt_b64.encode("ascii")) - expected = base64.b64decode(digest_b64.encode("ascii")) - except Exception: - return False - actual = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) - return hmac.compare_digest(actual, expected) diff --git a/src/govoplan_core/security/permissions.py b/src/govoplan_core/security/permissions.py index 64bd4d2..22052d4 100644 --- a/src/govoplan_core/security/permissions.py +++ b/src/govoplan_core/security/permissions.py @@ -86,6 +86,7 @@ SYSTEM_PERMISSIONS: tuple[PermissionDefinition, ...] = ( PermissionDefinition("system:audit:read", "View system audit", "Read audit records across tenants.", "System administration", "system"), PermissionDefinition("system:settings:read", "View system settings", "Read instance defaults and tenant-governance defaults.", "System administration", "system"), PermissionDefinition("system:settings:write", "Manage system settings", "Change instance defaults and tenant-governance defaults.", "System administration", "system"), + PermissionDefinition("system:maintenance:access", "Access during maintenance", "Use the system while maintenance mode is active.", "System administration", "system"), PermissionDefinition("system:governance:read", "View governance templates", "Inspect centrally managed group and role definitions.", "System administration", "system"), PermissionDefinition("system:governance:write", "Manage governance templates", "Create and assign centrally managed group and role definitions.", "System administration", "system"), ) @@ -174,6 +175,14 @@ DEFAULT_SYSTEM_ROLES: dict[str, dict[str, object]] = { "is_assignable": True, "managed": False, }, + "maintenance_operator": { + "name": "Maintenance operator", + "description": "Access the system while maintenance mode is active.", + "permissions": ["system:settings:read", "system:maintenance:access"], + "is_builtin": False, + "is_assignable": True, + "managed": False, + }, } @@ -186,8 +195,10 @@ def scope_grants(granted: str, required: str) -> bool: return True if required in LEGACY_SCOPE_ALIASES.get(granted, frozenset()): return True - if granted in {"*", "tenant:*"}: - return required in {"*", "tenant:*"} or (required in TENANT_SCOPES) or required in {"attachments:read", "attachments:write"} + if granted == "*": + return True + if granted == "tenant:*": + return required == "tenant:*" or not required.startswith("system:") if granted == "system:*": return required.startswith("system:") if granted.endswith(":*") and required.startswith(granted[:-1]): diff --git a/src/govoplan_core/security/sessions.py b/src/govoplan_core/security/sessions.py deleted file mode 100644 index 3ce6681..0000000 --- a/src/govoplan_core/security/sessions.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import timedelta - -from sqlalchemy.orm import Session - -from govoplan_core.db.models import ( - Account, - AuthSession, - Group, - GroupRoleAssignment, - Role, - SystemRoleAssignment, - Tenant, - User, - UserGroupMembership, - UserRoleAssignment, -) -from govoplan_core.access.auth.tokens import generate_secret, hash_secret, verify_secret -from govoplan_core.security.permissions import expand_scopes -from govoplan_core.security.time import ensure_aware_utc, utc_now - -SESSION_RANDOM_BYTES = 32 -DEFAULT_SESSION_HOURS = 12 - - -@dataclass(slots=True) -class CreatedSession: - model: AuthSession - token: str - csrf_token: str - - -def generate_session_token() -> str: - return generate_secret("ms_", random_bytes=SESSION_RANDOM_BYTES) - - -def hash_session_token(token: str) -> str: - return hash_secret(token) - - -def generate_csrf_token() -> str: - return generate_secret("", random_bytes=SESSION_RANDOM_BYTES) - - -def hash_csrf_token(token: str) -> str: - return hash_secret(token) - - -def verify_session_token(token: str, expected_hash: str) -> bool: - return verify_secret(token, expected_hash) - - -def verify_auth_session_csrf(auth_session: AuthSession, csrf_token: str | None) -> bool: - if not csrf_token or not auth_session.csrf_token_hash: - return False - return verify_secret(csrf_token, auth_session.csrf_token_hash) - - -def create_auth_session( - session: Session, - *, - user: User, - hours: int = DEFAULT_SESSION_HOURS, - user_agent: str | None = None, - ip_address: str | None = None, -) -> CreatedSession: - if not user.account_id: - raise ValueError("Tenant membership has no global account.") - token = generate_session_token() - csrf_token = generate_csrf_token() - now = utc_now() - model = AuthSession( - tenant_id=user.tenant_id, - user_id=user.id, - account_id=user.account_id, - token_hash=hash_session_token(token), - csrf_token_hash=hash_csrf_token(csrf_token), - expires_at=now + timedelta(hours=hours), - last_seen_at=now, - user_agent=user_agent, - ip_address=ip_address, - ) - user.last_login_at = now - if user.account: - user.account.last_login_at = now - session.add(user.account) - session.add(model) - session.add(user) - session.flush() - return CreatedSession(model=model, token=token, csrf_token=csrf_token) - - -def authenticate_session_token(session: Session, token: str) -> AuthSession | None: - token_hash = hash_session_token(token) - model = session.query(AuthSession).filter(AuthSession.token_hash == token_hash, AuthSession.revoked_at.is_(None)).one_or_none() - if not model: - return None - now = utc_now() - expires_at = ensure_aware_utc(model.expires_at) - if expires_at is None or expires_at < now: - return None - model.last_seen_at = now - session.add(model) - return model - - -def revoke_auth_session(session: Session, token: str) -> bool: - model = authenticate_session_token(session, token) - if not model: - return False - model.revoked_at = utc_now() - session.add(model) - return True - - -def switch_auth_session_tenant(session: Session, auth_session: AuthSession, tenant_id: str) -> User: - membership = ( - session.query(User) - .join(Tenant, Tenant.id == User.tenant_id) - .filter( - User.account_id == auth_session.account_id, - User.tenant_id == tenant_id, - User.is_active.is_(True), - Tenant.is_active.is_(True), - ) - .one_or_none() - ) - if membership is None: - raise LookupError("The account does not have an active membership in this tenant.") - auth_session.tenant_id = membership.tenant_id - auth_session.user_id = membership.id - auth_session.last_seen_at = utc_now() - session.add(auth_session) - session.flush() - return membership - - -def collect_direct_user_roles(session: Session, user: User) -> list[Role]: - return ( - session.query(Role) - .join(UserRoleAssignment, UserRoleAssignment.role_id == Role.id) - .filter( - UserRoleAssignment.user_id == user.id, - UserRoleAssignment.tenant_id == user.tenant_id, - Role.tenant_id == user.tenant_id, - ) - .order_by(Role.name.asc()) - .all() - ) - - -def collect_user_roles(session: Session, user: User) -> list[Role]: - roles_by_id: dict[str, Role] = {role.id: role for role in collect_direct_user_roles(session, user)} - - group_roles = ( - session.query(Role) - .join(GroupRoleAssignment, GroupRoleAssignment.role_id == Role.id) - .join(UserGroupMembership, UserGroupMembership.group_id == GroupRoleAssignment.group_id) - .join(Group, Group.id == UserGroupMembership.group_id) - .filter( - UserGroupMembership.user_id == user.id, - UserGroupMembership.tenant_id == user.tenant_id, - Group.is_active.is_(True), - Role.tenant_id == user.tenant_id, - ) - .all() - ) - for role in group_roles: - roles_by_id[role.id] = role - return list(roles_by_id.values()) - - -def collect_system_roles(session: Session, account: Account) -> list[Role]: - return ( - session.query(Role) - .join(SystemRoleAssignment, SystemRoleAssignment.role_id == Role.id) - .filter(SystemRoleAssignment.account_id == account.id, Role.tenant_id.is_(None)) - .order_by(Role.name.asc()) - .all() - ) - - -def collect_user_groups(session: Session, user: User) -> list[Group]: - return ( - session.query(Group) - .join(UserGroupMembership, UserGroupMembership.group_id == Group.id) - .filter( - UserGroupMembership.user_id == user.id, - UserGroupMembership.tenant_id == user.tenant_id, - Group.is_active.is_(True), - ) - .order_by(Group.name.asc()) - .all() - ) - - -def collect_user_scopes(session: Session, user: User, *, include_system: bool = True) -> list[str]: - scopes: set[str] = set() - for role in collect_user_roles(session, user): - scopes.update(role.permissions or []) - if include_system and user.account: - for role in collect_system_roles(session, user.account): - scopes.update(role.permissions or []) - return expand_scopes(scopes) - - -def collect_tenant_memberships(session: Session, account: Account) -> list[tuple[User, Tenant]]: - return ( - session.query(User, Tenant) - .join(Tenant, Tenant.id == User.tenant_id) - .filter( - User.account_id == account.id, - User.is_active.is_(True), - Tenant.is_active.is_(True), - ) - .order_by(Tenant.name.asc()) - .all() - ) diff --git a/src/govoplan_core/server/app.py b/src/govoplan_core/server/app.py index 98eb8e1..708714b 100644 --- a/src/govoplan_core/server/app.py +++ b/src/govoplan_core/server/app.py @@ -2,12 +2,13 @@ from __future__ import annotations from fastapi import APIRouter -from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.lifecycle import ModuleLifecycleManager +from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids from govoplan_core.db.session import configure_database from govoplan_core.server.config import GovoplanServerConfig, load_server_config from govoplan_core.server.fastapi import create_govoplan_app from govoplan_core.server.platform import create_platform_router -from govoplan_core.server.registry import build_platform_registry +from govoplan_core.server.registry import available_module_manifests, build_platform_registry def create_app(config: GovoplanServerConfig | str | None = None): @@ -20,17 +21,30 @@ def create_app(config: GovoplanServerConfig | str | None = None): if database_url: configure_database(str(database_url)) - registry = build_platform_registry(server_config.enabled_modules, manifest_factories=server_config.manifest_factories) + raw_enabled_modules = load_startup_enabled_modules(server_config.enabled_modules) + candidate_modules = startup_candidate_module_ids(server_config.enabled_modules, raw_enabled_modules) + startup_available_modules = available_module_manifests( + server_config.manifest_factories, + enabled_modules=candidate_modules, + ignore_load_errors=True, + ) + available_modules = available_module_manifests(server_config.manifest_factories, ignore_load_errors=True) + enabled_modules = load_startup_enabled_modules(server_config.enabled_modules, available=startup_available_modules) + registry = build_platform_registry(enabled_modules, manifest_factories=server_config.manifest_factories) api_router = APIRouter(prefix=server_config.api_prefix) for router in server_config.base_routers: api_router.include_router(router) - module_context = ModuleContext(registry=registry, settings=server_config.settings, data=server_config.module_context_data) - registry.configure_capability_context(module_context) - for manifest in registry.manifests(): - if manifest.route_factory is not None: - api_router.include_router(manifest.route_factory(module_context)) + lifecycle = ModuleLifecycleManager( + registry=registry, + available_modules=available_modules, + settings=server_config.settings, + api_prefix=server_config.api_prefix, + manifest_factories=tuple(server_config.manifest_factories), + module_context_data=server_config.module_context_data, + ) + lifecycle.configure_runtime() api_router.include_router(create_platform_router(settings=server_config.settings)) @@ -49,6 +63,8 @@ def create_app(config: GovoplanServerConfig | str | None = None): lifespan=server_config.lifespan, cors_origins=server_config.cors_origins, ) + lifecycle.attach_app(app) + lifecycle.apply_enabled_modules(tuple(manifest.id for manifest in registry.manifests()), migrate=False) for configurator in server_config.app_configurators: configurator(app, registry, server_config.settings) return app diff --git a/src/govoplan_core/server/default_config.py b/src/govoplan_core/server/default_config.py index ea12176..c1ae028 100644 --- a/src/govoplan_core/server/default_config.py +++ b/src/govoplan_core/server/default_config.py @@ -1,20 +1,23 @@ from __future__ import annotations from contextlib import asynccontextmanager -from importlib import import_module from fastapi import Depends, FastAPI -from govoplan_core.auth.dependencies import ApiPrincipal, require_scope +from govoplan_access.backend.auth.dependencies import ApiPrincipal, require_scope from govoplan_core.core.registry import PlatformRegistry from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables from govoplan_core.db.session import get_database -from govoplan_core.server.config import GovoplanServerConfig, RouterContribution +from govoplan_core.server.config import GovoplanServerConfig from govoplan_core.settings import Settings, settings @asynccontextmanager async def lifespan(app: FastAPI): + if settings.app_env.lower() == "dev" and settings.dev_auto_migrate_enabled: + from govoplan_core.db.migrations import migrate_database + + migrate_database(database_url=settings.database_url) if settings.app_env.lower() == "dev" and settings.dev_bootstrap_enabled: create_all_tables() with get_database().SessionLocal() as session: @@ -30,22 +33,6 @@ def _cors_origins(value: str) -> list[str]: return [item.strip() for item in value.split(",") if item.strip()] -def _settings_module_enabled(module_id: str) -> bool: - return module_id in {item.strip() for item in settings.enabled_modules.split(",") if item.strip()} - - -def _dev_mail_enabled(config_settings: object | None, registry: PlatformRegistry) -> bool: - active_settings = config_settings if isinstance(config_settings, Settings) else settings - return active_settings.app_env == "dev" and active_settings.dev_mailbox_api_enabled and registry.has_module("mail") - - -def _dev_mail_router_contributions() -> tuple[RouterContribution, ...]: - if not (settings.app_env == "dev" and settings.dev_mailbox_api_enabled and _settings_module_enabled("mail")): - return () - module = import_module("govoplan_mail.backend.dev_router") - return (RouterContribution(module.router, required_module="mail", enabled=_dev_mail_enabled),) - - def register_health_details(app: FastAPI, registry: PlatformRegistry, config_settings: object | None) -> None: active_settings = config_settings if isinstance(config_settings, Settings) else settings @@ -70,19 +57,12 @@ def register_health_details(app: FastAPI, registry: PlatformRegistry, config_set def get_server_config() -> GovoplanServerConfig: - from govoplan_core.api.v1.admin import router as admin_router - from govoplan_core.api.v1.audit import router as audit_router - from govoplan_core.api.v1.auth import router as auth_router - from govoplan_core.api.v1.system import router as system_router - return GovoplanServerConfig( title="GovOPlaN Server", version="0.3.0", settings=settings, enabled_modules=settings.enabled_modules, api_prefix="/api/v1", - base_routers=(auth_router, admin_router, audit_router, system_router), - extra_routers=_dev_mail_router_contributions(), lifespan=lifespan, cors_origins=_cors_origins(settings.cors_origins), app_configurators=(register_health_details,), diff --git a/src/govoplan_core/server/platform.py b/src/govoplan_core/server/platform.py index d26d907..b6e4284 100644 --- a/src/govoplan_core/server/platform.py +++ b/src/govoplan_core/server/platform.py @@ -1,9 +1,12 @@ from __future__ import annotations from fastapi import APIRouter, HTTPException, Request +from sqlalchemy.exc import SQLAlchemyError +from govoplan_core.core.maintenance import saved_maintenance_mode from govoplan_core.core.modules import FrontendModule, FrontendRoute, NavItem from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.session import get_database def _registry(request: Request) -> PlatformRegistry: @@ -42,6 +45,10 @@ def _frontend_payload(frontend: FrontendModule | None) -> dict[str, object] | No "module_id": frontend.module_id, "package_name": frontend.package_name, "asset_manifest": frontend.asset_manifest, + "asset_manifest_signature": frontend.asset_manifest_signature, + "asset_manifest_public_key_id": frontend.asset_manifest_public_key_id, + "asset_manifest_integrity": frontend.asset_manifest_integrity, + "asset_manifest_contract_version": frontend.asset_manifest_contract_version, "routes": [_frontend_route_payload(route) for route in frontend.routes], "nav": [_nav_item_payload(item) for item in frontend.nav_items], "settings_routes": [_frontend_route_payload(route) for route in frontend.settings_routes], @@ -60,6 +67,17 @@ def _runtime_ui_capabilities(manifest_id: str, settings: object | None, registry def create_platform_router(settings: object | None = None) -> APIRouter: router = APIRouter(prefix="/platform", tags=["platform"]) + @router.get("/status") + def status(): + try: + with get_database().session() as session: + maintenance_mode = saved_maintenance_mode(session) + except (RuntimeError, SQLAlchemyError): + maintenance_mode = None + return { + "maintenance_mode": maintenance_mode.as_dict() if maintenance_mode is not None else {"enabled": False, "message": None}, + } + @router.get("/modules") def modules(request: Request): registry = _registry(request) diff --git a/src/govoplan_core/server/registry.py b/src/govoplan_core/server/registry.py index d4e668b..69e91c5 100644 --- a/src/govoplan_core/server/registry.py +++ b/src/govoplan_core/server/registry.py @@ -2,10 +2,11 @@ from __future__ import annotations from collections.abc import Callable, Iterable, Sequence -from govoplan_core.access.manifest import get_manifest as get_access_manifest +from govoplan_access.backend.manifest import get_manifest as get_access_manifest from govoplan_core.core.discovery import discover_module_manifests from govoplan_core.core.modules import ModuleManifest from govoplan_core.core.registry import PlatformRegistry, RegistryError +from govoplan_tenancy.backend.manifest import get_manifest as get_tenancy_manifest ManifestFactory = Callable[[], ModuleManifest] @@ -18,8 +19,20 @@ def parse_enabled_modules(value: str | Iterable[str]) -> list[str]: return [item for item in items if item] -def available_module_manifests(manifest_factories: Sequence[ManifestFactory] = (), *, enabled_modules: Iterable[str] | None = None) -> dict[str, ModuleManifest]: - manifests = {manifest.id: manifest for manifest in discover_module_manifests(enabled_modules=enabled_modules)} +def available_module_manifests( + manifest_factories: Sequence[ManifestFactory] = (), + *, + enabled_modules: Iterable[str] | None = None, + ignore_load_errors: bool = False, +) -> dict[str, ModuleManifest]: + manifests = { + manifest.id: manifest + for manifest in discover_module_manifests( + enabled_modules=enabled_modules, + ignore_load_errors=ignore_load_errors, + ) + } + manifests.setdefault("tenancy", get_tenancy_manifest()) manifests.setdefault("access", get_access_manifest()) for factory in manifest_factories: manifest = factory() @@ -29,8 +42,15 @@ def available_module_manifests(manifest_factories: Sequence[ManifestFactory] = ( def build_platform_registry(enabled_modules: str | Iterable[str], *, manifest_factories: Sequence[ManifestFactory] = ()) -> PlatformRegistry: requested = parse_enabled_modules(enabled_modules) + if "tenancy" not in requested: + requested.insert(0, "tenancy") if "access" not in requested: - requested.insert(0, "access") + tenancy_index = requested.index("tenancy") + requested.insert(tenancy_index + 1, "access") + elif requested.index("access") < requested.index("tenancy"): + requested.remove("access") + tenancy_index = requested.index("tenancy") + requested.insert(tenancy_index + 1, "access") available = available_module_manifests(manifest_factories, enabled_modules=requested) registry = PlatformRegistry() diff --git a/src/govoplan_core/settings.py b/src/govoplan_core/settings.py index e041f4b..e0cb637 100644 --- a/src/govoplan_core/settings.py +++ b/src/govoplan_core/settings.py @@ -17,7 +17,7 @@ class Settings(BaseSettings): tenant_data_mode: str = Field(default="shared", alias="TENANT_DATA_MODE") tenant_db_url_template: str | None = Field(default=None, alias="TENANT_DB_URL_TEMPLATE") tenant_schema_template: str | None = Field(default=None, alias="TENANT_SCHEMA_TEMPLATE") - enabled_modules: str = Field(default="access,campaigns,files,mail", alias="ENABLED_MODULES") + enabled_modules: str = Field(default="tenancy,access,admin,policy,audit,campaigns,files,mail,calendar", alias="ENABLED_MODULES") redis_url: str = Field(default="redis://redis:6379/0", alias="REDIS_URL") celery_enabled: bool = Field(default=False, alias="CELERY_ENABLED") @@ -53,6 +53,7 @@ class Settings(BaseSettings): # Development bootstrap only. Do not use this in production. dev_bootstrap_api_key: str | None = Field(default="dev-multimailer-api-key", alias="DEV_BOOTSTRAP_API_KEY") + dev_auto_migrate_enabled: bool = Field(default=True, alias="DEV_AUTO_MIGRATE_ENABLED") dev_bootstrap_enabled: bool = Field(default=False, alias="DEV_BOOTSTRAP_ENABLED") dev_bootstrap_password: str = Field(default="dev-admin", alias="DEV_BOOTSTRAP_PASSWORD") dev_mailbox_api_enabled: bool = Field(default=False, alias="DEV_MAILBOX_API_ENABLED") diff --git a/src/govoplan_core/tenancy/__init__.py b/src/govoplan_core/tenancy/__init__.py new file mode 100644 index 0000000..f6ba620 --- /dev/null +++ b/src/govoplan_core/tenancy/__init__.py @@ -0,0 +1,2 @@ +"""Tenant administration helpers shared by platform route modules.""" + diff --git a/src/govoplan_core/tenancy/service.py b/src/govoplan_core/tenancy/service.py new file mode 100644 index 0000000..5ac21ef --- /dev/null +++ b/src/govoplan_core/tenancy/service.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlalchemy.orm import Session + +from govoplan_core.admin.common import AdminValidationError +from govoplan_core.admin.settings import get_system_settings +from govoplan_core.core.runtime import get_registry +from govoplan_access.backend.db.models import ApiKey, Group, User +from govoplan_tenancy.backend.db.models import Tenant + + +@dataclass(frozen=True) +class EffectiveTenantGovernance: + allow_custom_groups: bool + allow_custom_roles: bool + allow_api_keys: bool + + +def _narrowing_bool(system_allows: bool, tenant_override: bool | None) -> bool: + if not system_allows: + return False + return tenant_override is not False + + +def effective_tenant_governance(session: Session, tenant: Tenant) -> EffectiveTenantGovernance: + defaults = get_system_settings(session) + return EffectiveTenantGovernance( + allow_custom_groups=_narrowing_bool(defaults.allow_tenant_custom_groups, tenant.allow_custom_groups), + allow_custom_roles=_narrowing_bool(defaults.allow_tenant_custom_roles, tenant.allow_custom_roles), + allow_api_keys=_narrowing_bool(defaults.allow_tenant_api_keys, tenant.allow_api_keys), + ) + + +def assert_tenant_governance_override_allowed(session: Session, *, field: str, value: bool | None) -> None: + if value is not True: + return + defaults = get_system_settings(session) + blocked = { + "allow_custom_groups": not defaults.allow_tenant_custom_groups, + "allow_custom_roles": not defaults.allow_tenant_custom_roles, + "allow_api_keys": not defaults.allow_tenant_api_keys, + } + if blocked.get(field): + raise AdminValidationError("Tenant governance cannot explicitly allow a capability denied by system settings.") + + +def _tenant_module_counts(session: Session, tenant_id: str) -> dict[str, int]: + registry = get_registry() + if registry is None or not hasattr(registry, "tenant_summary_providers"): + return {} + counts: dict[str, int] = {} + for provider in registry.tenant_summary_providers().values(): + provided = provider(session, tenant_id) + counts.update({str(key): int(value) for key, value in provided.items()}) + return counts + + +def tenant_counts(session: Session, tenant_id: str) -> dict[str, int]: + module_counts = _tenant_module_counts(session, tenant_id) + + return { + "users": session.query(User).filter(User.tenant_id == tenant_id).count(), + "active_users": session.query(User).filter(User.tenant_id == tenant_id, User.is_active.is_(True)).count(), + "groups": session.query(Group).filter(Group.tenant_id == tenant_id).count(), + "campaigns": module_counts.get("campaigns", 0), + "files": module_counts.get("files", 0), + "api_keys": session.query(ApiKey).filter(ApiKey.tenant_id == tenant_id, ApiKey.revoked_at.is_(None)).count(), + } diff --git a/tests/test_access_contracts.py b/tests/test_access_contracts.py new file mode 100644 index 0000000..ac0a04b --- /dev/null +++ b/tests/test_access_contracts.py @@ -0,0 +1,577 @@ +from __future__ import annotations + +import shutil +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +from fastapi import APIRouter, Depends +from fastapi.testclient import TestClient + +from govoplan_access.backend.auth.dependencies import ApiPrincipal, get_api_principal +from govoplan_core.core.access import ( + ACCESS_CAPABILITY_NAMES, + ACCESS_MODULE_ID, + CAPABILITY_ACCESS_ADMINISTRATION, + CAPABILITY_ACCESS_DIRECTORY, + CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER, + CAPABILITY_ACCESS_PERMISSION_EVALUATOR, + CAPABILITY_ACCESS_PRINCIPAL_RESOLVER, + CAPABILITY_ACCESS_RESOURCE_ACCESS, + CAPABILITY_ACCESS_TENANT_PROVISIONER, + CAPABILITY_AUDIT_SINK, + CAPABILITY_SECURITY_SECRET_PROVIDER, + CAPABILITY_TENANCY_TENANT_RESOLVER, + AccessDecision, + AccessAdministration, + AccessDirectory, + AccessGovernanceMaterializer, + AccessSubjectRef, + AccountRef, + AuditEvent, + AuditSink, + GovernanceTemplateMaterialization, + GroupRef, + PermissionEvaluator, + PrincipalRef, + PrincipalResolver, + ResourceAccessService, + RoleRef, + SecretProvider, + TenantRef, + TenantAccessProvisioner, + TenantOwnerCandidateRef, + TenantResolver, + UserRef, +) +from govoplan_core.core.campaigns import ( + CAPABILITY_CAMPAIGNS_ACCESS, + CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, + CAPABILITY_CAMPAIGNS_MAIL_POLICY_CONTEXT, + CAPABILITY_CAMPAIGNS_POLICY_CONTEXT, + CAPABILITY_CAMPAIGNS_RETENTION, + CampaignAccessProvider, + CampaignDeliveryTaskProvider, + CampaignMailPolicyContext, + CampaignMailPolicyContextProvider, + CampaignPolicyContext, + CampaignPolicyContextProvider, + CampaignRetentionProvider, +) +from govoplan_core.core.modules import ModuleContext, ModuleManifest +from govoplan_core.core.registry import PlatformRegistry +from govoplan_core.db.base import Base +from govoplan_core.db.session import configure_database, get_database +from govoplan_core.server.app import create_app +from govoplan_core.server.config import GovoplanServerConfig +from govoplan_access.backend.db.models import Account, Group, User, UserGroupMembership +from govoplan_tenancy.backend.db.models import Tenant + + +class _FakeAccessDirectory: + account = AccountRef(id="account-1", email="demo@example.test", display_name="Demo") + user = UserRef(id="user-1", account_id=account.id, tenant_id="tenant-1", email=account.email) + group = GroupRef(id="group-1", tenant_id="tenant-1", name="Team") + + def get_account(self, account_id: str) -> AccountRef | None: + return self.account if account_id == self.account.id else None + + def get_user(self, user_id: str) -> UserRef | None: + return self.user if user_id == self.user.id else None + + def get_users(self, user_ids): + return {self.user.id: self.user} if self.user.id in set(user_ids) else {} + + def users_for_tenant(self, tenant_id: str): + return (self.user,) if tenant_id == self.user.tenant_id else () + + def get_group(self, group_id: str) -> GroupRef | None: + return self.group if group_id == self.group.id else None + + def get_groups(self, group_ids): + return {self.group.id: self.group} if self.group.id in set(group_ids) else {} + + def groups_for_tenant(self, tenant_id: str): + return (self.group,) if tenant_id == self.group.tenant_id else () + + def groups_for_user(self, user_id: str, *, tenant_id: str): + if user_id == self.user.id and tenant_id == self.user.tenant_id: + return (self.group,) + return () + + def display_label(self, subject: AccessSubjectRef) -> str | None: + if subject.kind == "user" and subject.id == self.user.id: + return self.user.display_name or self.user.email + if subject.kind == "group" and subject.id == self.group.id: + return self.group.name + return subject.label + + +class _FakePrincipalResolver: + def resolve_request(self, request: object, *, session: object | None = None) -> PrincipalRef: + del request, session + return PrincipalRef( + account_id="account-1", + membership_id="user-1", + tenant_id="tenant-1", + scopes=frozenset({"files:file:read"}), + group_ids=frozenset({"group-1"}), + ) + + +class _FakeTenantResolver: + tenant = TenantRef(id="tenant-1", name="Tenant") + + def get_tenant(self, tenant_id: str) -> TenantRef | None: + return self.tenant if tenant_id == self.tenant.id else None + + def current_tenant(self, principal: PrincipalRef) -> TenantRef | None: + return self.get_tenant(principal.tenant_id or "") + + +class _FakePermissionEvaluator: + def scopes_grant(self, scopes, required_scope: str) -> bool: + return required_scope in set(scopes) + + def has_scope(self, principal: PrincipalRef, required_scope: str) -> bool: + return self.scopes_grant(principal.scopes, required_scope) + + def explain_scope(self, principal: PrincipalRef, required_scope: str) -> AccessDecision: + allowed = self.has_scope(principal, required_scope) + return AccessDecision(allowed=allowed, reason=None if allowed else "missing scope", requirements=(required_scope,)) + + +class _FakeResourceAccessService: + def explain(self, principal: PrincipalRef, *, resource_type: str, resource_id: str, action: str) -> AccessDecision: + del principal, resource_type, resource_id, action + return AccessDecision(allowed=True) + + +class _FakeTenantAccessProvisioner: + def ensure_default_roles(self, session: object, tenant: object | None = None): + del session, tenant + return {} + + def tenant_owner_candidates(self, session: object): + del session + return (TenantOwnerCandidateRef(account_id="account-1", email="demo@example.test", display_name="Demo"),) + + def ensure_tenant_owner_membership(self, session: object, *, tenant: object, owner_account_id: str) -> str: + del session, tenant, owner_account_id + return "user-1" + + +class _FakeAccessAdministration: + def system_account_count(self, session: object) -> int: + del session + return 1 + + def role_count_for_tenant(self, session: object, tenant_id: str) -> int: + del session, tenant_id + return 2 + + def active_api_key_count_for_tenant(self, session: object, tenant_id: str) -> int: + del session, tenant_id + return 3 + + def actor_email_by_user_id(self, session: object, user_ids): + del session + return {user_id: "demo@example.test" for user_id in user_ids} + + def user_ids_for_actor_filter(self, session: object, *, operator: str, value: str): + del session, operator, value + return ("user-1",) + + +class _FakeAccessGovernanceMaterializer: + def __init__(self) -> None: + self.synced: list[GovernanceTemplateMaterialization] = [] + self.removed: list[GovernanceTemplateMaterialization] = [] + + def sync_template(self, session: object, template: GovernanceTemplateMaterialization) -> None: + del session + self.synced.append(template) + + def remove_template(self, session: object, template: GovernanceTemplateMaterialization) -> None: + del session + self.removed.append(template) + + +class _FakeCampaignMailPolicyContextProvider: + context = CampaignMailPolicyContext( + id="campaign-1", + tenant_id="tenant-1", + owner_user_id="user-1", + owner_group_id="group-1", + mail_profile_policy={"allow_campaign_profiles": False}, + ) + + def get_campaign_mail_policy_context(self, session: object, *, tenant_id: str, campaign_id: str): + del session + if tenant_id == self.context.tenant_id and campaign_id == self.context.id: + return self.context + return None + + def set_campaign_mail_profile_policy(self, session: object, *, tenant_id: str, campaign_id: str, policy): + del session + if tenant_id != self.context.tenant_id or campaign_id != self.context.id: + raise ValueError("Campaign policy not found") + return dict(policy) + + +class _FakeCampaignAccessProvider: + def campaign_exists(self, session: object, *, tenant_id: str, campaign_id: str) -> bool: + del session + return tenant_id == "tenant-1" and campaign_id == "campaign-1" + + def can_read_campaign( + self, + session: object, + *, + tenant_id: str, + campaign_id: str, + user_id: str, + group_ids=(), + tenant_admin: bool = False, + ) -> bool: + del session + return self.campaign_exists(object(), tenant_id=tenant_id, campaign_id=campaign_id) and ( + tenant_admin or user_id == "user-1" or "group-1" in set(group_ids) + ) + + +class _FakeCampaignPolicyContextProvider: + context = CampaignPolicyContext( + id="campaign-1", + tenant_id="tenant-1", + owner_user_id="user-1", + owner_group_id="group-1", + settings={"privacy_retention_policy": {"audit_detail_level": "redacted"}}, + ) + + def get_campaign_policy_context(self, session: object, *, campaign_id: str, tenant_id: str | None = None): + del session + if campaign_id != self.context.id or (tenant_id is not None and tenant_id != self.context.tenant_id): + return None + return self.context + + def set_campaign_settings(self, session: object, *, tenant_id: str, campaign_id: str, settings): + del session + if tenant_id != self.context.tenant_id or campaign_id != self.context.id: + raise ValueError("Campaign privacy policy not found") + return dict(settings) + + +class _FakeCampaignDeliveryTaskProvider: + def send_campaign_job(self, session: object, *, job_id: str, enqueue_imap_task: bool = True): + del session + return {"job_id": job_id, "enqueue_imap_task": enqueue_imap_task} + + def append_sent_for_job(self, session: object, *, job_id: str): + del session + return {"job_id": job_id, "status": "appended"} + + +class _FakeCampaignRetentionProvider: + def apply_retention(self, session: object, *, dry_run: bool, now, policy_for_campaign_id): + del session, now + policy_for_campaign_id("campaign-1") + return {"raw_campaign_json": {"eligible": int(dry_run)}} + + +class _FakeSecretProvider: + def __init__(self) -> None: + self._values: dict[str, str] = {} + + def store_secret(self, *, scope: str, name: str, value: str) -> str: + ref = f"{scope}/{name}" + self._values[ref] = value + return ref + + def read_secret(self, secret_ref: str) -> str | None: + return self._values.get(secret_ref) + + def delete_secret(self, secret_ref: str) -> None: + self._values.pop(secret_ref, None) + + +class _FakeAuditSink: + def __init__(self) -> None: + self.events: list[AuditEvent] = [] + + def record(self, event: AuditEvent) -> None: + self.events.append(event) + + +class AccessContractTests(unittest.TestCase): + def test_access_capability_names_are_stable(self) -> None: + self.assertEqual("access", ACCESS_MODULE_ID) + self.assertEqual("access.principalResolver", CAPABILITY_ACCESS_PRINCIPAL_RESOLVER) + self.assertEqual("access.directory", CAPABILITY_ACCESS_DIRECTORY) + self.assertEqual("access.permissionEvaluator", CAPABILITY_ACCESS_PERMISSION_EVALUATOR) + self.assertEqual("access.resourceAccess", CAPABILITY_ACCESS_RESOURCE_ACCESS) + self.assertEqual("access.tenantProvisioner", CAPABILITY_ACCESS_TENANT_PROVISIONER) + self.assertEqual("access.administration", CAPABILITY_ACCESS_ADMINISTRATION) + self.assertEqual("access.governanceMaterializer", CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER) + self.assertEqual("campaigns.access", CAPABILITY_CAMPAIGNS_ACCESS) + self.assertEqual("campaigns.deliveryTasks", CAPABILITY_CAMPAIGNS_DELIVERY_TASKS) + self.assertEqual("campaigns.mailPolicyContext", CAPABILITY_CAMPAIGNS_MAIL_POLICY_CONTEXT) + self.assertEqual("campaigns.policyContext", CAPABILITY_CAMPAIGNS_POLICY_CONTEXT) + self.assertEqual("campaigns.retention", CAPABILITY_CAMPAIGNS_RETENTION) + self.assertEqual("tenancy.tenantResolver", CAPABILITY_TENANCY_TENANT_RESOLVER) + self.assertEqual("security.secretProvider", CAPABILITY_SECURITY_SECRET_PROVIDER) + self.assertEqual("audit.sink", CAPABILITY_AUDIT_SINK) + self.assertEqual(len(ACCESS_CAPABILITY_NAMES), len(set(ACCESS_CAPABILITY_NAMES))) + + def test_access_refs_are_small_immutable_dtos(self) -> None: + principal = PrincipalRef( + account_id="account-1", + membership_id="user-1", + tenant_id="tenant-1", + scopes=frozenset({"campaigns:campaign:read"}), + group_ids=frozenset({"group-1"}), + email="demo@example.test", + ) + role = RoleRef(id="role-1", name="Reviewer", level="tenant", tenant_id="tenant-1") + + self.assertEqual("account-1", principal.account_id) + self.assertEqual(frozenset({"campaigns:campaign:read"}), principal.scopes) + self.assertEqual("Reviewer", role.name) + with self.assertRaises(AttributeError): + principal.account_id = "changed" # type: ignore[misc] + + def test_protocol_shapes_match_reference_implementations(self) -> None: + self.assertIsInstance(_FakePrincipalResolver(), PrincipalResolver) + self.assertIsInstance(_FakeTenantResolver(), TenantResolver) + self.assertIsInstance(_FakeAccessDirectory(), AccessDirectory) + self.assertIsInstance(_FakePermissionEvaluator(), PermissionEvaluator) + self.assertIsInstance(_FakeResourceAccessService(), ResourceAccessService) + self.assertIsInstance(_FakeTenantAccessProvisioner(), TenantAccessProvisioner) + self.assertIsInstance(_FakeAccessAdministration(), AccessAdministration) + self.assertIsInstance(_FakeAccessGovernanceMaterializer(), AccessGovernanceMaterializer) + self.assertIsInstance(_FakeCampaignAccessProvider(), CampaignAccessProvider) + self.assertIsInstance(_FakeCampaignDeliveryTaskProvider(), CampaignDeliveryTaskProvider) + self.assertIsInstance(_FakeCampaignMailPolicyContextProvider(), CampaignMailPolicyContextProvider) + self.assertIsInstance(_FakeCampaignPolicyContextProvider(), CampaignPolicyContextProvider) + self.assertIsInstance(_FakeCampaignRetentionProvider(), CampaignRetentionProvider) + self.assertIsInstance(_FakeSecretProvider(), SecretProvider) + self.assertIsInstance(_FakeAuditSink(), AuditSink) + + def test_campaign_mail_policy_context_capability_shape(self) -> None: + provider = _FakeCampaignMailPolicyContextProvider() + context = provider.get_campaign_mail_policy_context(object(), tenant_id="tenant-1", campaign_id="campaign-1") + + self.assertEqual("campaign-1", context.id) # type: ignore[union-attr] + self.assertEqual("user-1", context.owner_user_id) # type: ignore[union-attr] + self.assertEqual({"allow_campaign_profiles": False}, context.mail_profile_policy) # type: ignore[union-attr] + self.assertEqual({"allow_campaign_profiles": True}, provider.set_campaign_mail_profile_policy(object(), tenant_id="tenant-1", campaign_id="campaign-1", policy={"allow_campaign_profiles": True})) + + def test_campaign_access_capability_shape(self) -> None: + provider = _FakeCampaignAccessProvider() + + self.assertTrue(provider.campaign_exists(object(), tenant_id="tenant-1", campaign_id="campaign-1")) + self.assertTrue(provider.can_read_campaign(object(), tenant_id="tenant-1", campaign_id="campaign-1", user_id="other", group_ids=("group-1",))) + self.assertFalse(provider.can_read_campaign(object(), tenant_id="tenant-1", campaign_id="missing", user_id="user-1")) + + def test_campaign_policy_delivery_and_retention_capability_shapes(self) -> None: + policy_provider = _FakeCampaignPolicyContextProvider() + delivery_provider = _FakeCampaignDeliveryTaskProvider() + retention_provider = _FakeCampaignRetentionProvider() + + context = policy_provider.get_campaign_policy_context(object(), tenant_id="tenant-1", campaign_id="campaign-1") + self.assertEqual("tenant-1", context.tenant_id) # type: ignore[union-attr] + self.assertEqual({"privacy_retention_policy": {"audit_detail_level": "redacted"}}, context.settings) # type: ignore[union-attr] + self.assertEqual({"demo": True}, policy_provider.set_campaign_settings(object(), tenant_id="tenant-1", campaign_id="campaign-1", settings={"demo": True})) + self.assertEqual({"job_id": "job-1", "enqueue_imap_task": False}, delivery_provider.send_campaign_job(object(), job_id="job-1", enqueue_imap_task=False)) + self.assertEqual({"job_id": "job-1", "status": "appended"}, delivery_provider.append_sent_for_job(object(), job_id="job-1")) + self.assertEqual({"raw_campaign_json": {"eligible": 1}}, retention_provider.apply_retention(object(), dry_run=True, now=object(), policy_for_campaign_id=lambda campaign_id: object())) + + def test_access_capabilities_register_and_resolve_through_platform_registry(self) -> None: + directory = _FakeAccessDirectory() + resolver = _FakePrincipalResolver() + evaluator = _FakePermissionEvaluator() + tenant_resolver = _FakeTenantResolver() + administration = _FakeAccessAdministration() + materializer = _FakeAccessGovernanceMaterializer() + manifest = ModuleManifest( + id=ACCESS_MODULE_ID, + name="Access", + version="test", + capability_factories={ + CAPABILITY_ACCESS_DIRECTORY: lambda context: directory, + CAPABILITY_ACCESS_PRINCIPAL_RESOLVER: lambda context: resolver, + CAPABILITY_ACCESS_PERMISSION_EVALUATOR: lambda context: evaluator, + CAPABILITY_ACCESS_ADMINISTRATION: lambda context: administration, + CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER: lambda context: materializer, + CAPABILITY_TENANCY_TENANT_RESOLVER: lambda context: tenant_resolver, + }, + ) + + registry = PlatformRegistry() + registry.register(manifest) + registry.configure_capability_context(ModuleContext(registry=registry, settings=object())) + + self.assertIs(directory, registry.require_capability(CAPABILITY_ACCESS_DIRECTORY)) + self.assertIs(resolver, registry.require_capability(CAPABILITY_ACCESS_PRINCIPAL_RESOLVER)) + self.assertIs(evaluator, registry.require_capability(CAPABILITY_ACCESS_PERMISSION_EVALUATOR)) + self.assertIs(administration, registry.require_capability(CAPABILITY_ACCESS_ADMINISTRATION)) + self.assertIs(materializer, registry.require_capability(CAPABILITY_ACCESS_GOVERNANCE_MATERIALIZER)) + self.assertIs(tenant_resolver, registry.require_capability(CAPABILITY_TENANCY_TENANT_RESOLVER)) + + def test_sql_directory_and_tenant_resolver_return_access_dtos(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-access-directory-")) + try: + configure_database(f"sqlite:///{root / 'directory.db'}") + database = get_database() + Base.metadata.create_all(bind=database.engine) + with database.session() as session: + tenant = Tenant(id="tenant-directory", slug="directory", name="Directory Tenant", settings={}) + account = Account( + id="account-directory", + email="directory@example.test", + normalized_email="directory@example.test", + display_name="Directory User", + ) + user = User( + id="user-directory", + tenant_id=tenant.id, + account_id=account.id, + email=account.email, + display_name=None, + settings={}, + mail_profile_policy={}, + ) + group = Group(id="group-directory", tenant_id=tenant.id, slug="review", name="Review", description=None) + membership = UserGroupMembership(tenant_id=tenant.id, user_id=user.id, group_id=group.id) + session.add_all([tenant, account, user, group, membership]) + session.commit() + + from govoplan_access.backend.directory import SqlAccessDirectory + from govoplan_tenancy.backend.capabilities import SqlTenantResolver + + directory = SqlAccessDirectory() + tenant_resolver = SqlTenantResolver() + + self.assertEqual("directory@example.test", directory.get_account("account-directory").email) # type: ignore[union-attr] + self.assertEqual("Directory User", directory.get_user("user-directory").display_name) # type: ignore[union-attr] + self.assertEqual(["user-directory"], [user.id for user in directory.users_for_tenant("tenant-directory")]) + self.assertEqual(["group-directory"], [group.id for group in directory.groups_for_tenant("tenant-directory")]) + self.assertEqual(["group-directory"], [group.id for group in directory.groups_for_user("user-directory", tenant_id="tenant-directory")]) + self.assertEqual("Review", directory.display_label(AccessSubjectRef(kind="group", id="group-directory", tenant_id="tenant-directory"))) + self.assertEqual("directory", tenant_resolver.get_tenant("tenant-directory").slug) # type: ignore[union-attr] + self.assertEqual( + "tenant-directory", + tenant_resolver.current_tenant( + PrincipalRef(account_id="account-directory", membership_id="user-directory", tenant_id="tenant-directory") + ).id, # type: ignore[union-attr] + ) + finally: + shutil.rmtree(root, ignore_errors=True) + + def test_api_principal_dependency_uses_access_resolver_capability(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-access-contract-")) + try: + account_id = "account-capability" + tenant_id = "tenant-capability" + user_id = "user-capability" + + class CapabilityPrincipalResolver: + def resolve_request(self, request: object, *, session: object | None = None) -> PrincipalRef: + del request, session + return PrincipalRef( + account_id=account_id, + membership_id=user_id, + tenant_id=tenant_id, + scopes=frozenset(), + ) + + class CapabilityPermissionEvaluator: + def scopes_grant(self, scopes, required_scope: str) -> bool: + del scopes + return required_scope == "demo:scope:read" + + def has_scope(self, principal: PrincipalRef, required_scope: str) -> bool: + del principal + return required_scope == "demo:scope:read" + + def explain_scope(self, principal: PrincipalRef, required_scope: str) -> AccessDecision: + return AccessDecision(allowed=self.has_scope(principal, required_scope), requirements=(required_scope,)) + + router = APIRouter() + + @router.get("/capability-principal") + def capability_principal(principal: ApiPrincipal = Depends(get_api_principal)) -> dict[str, object]: + return { + "account_id": principal.account_id, + "tenant_id": principal.tenant_id, + "user_id": principal.user.id, + "has_demo_scope": principal.has("demo:scope:read"), + } + + def access_manifest() -> ModuleManifest: + return ModuleManifest( + id=ACCESS_MODULE_ID, + name="Access", + version="test", + capability_factories={ + CAPABILITY_ACCESS_PRINCIPAL_RESOLVER: lambda context: CapabilityPrincipalResolver(), + CAPABILITY_ACCESS_PERMISSION_EVALUATOR: lambda context: CapabilityPermissionEvaluator(), + }, + ) + + config = GovoplanServerConfig( + title="access contract test", + version="test", + settings=SimpleNamespace(database_url=f"sqlite:///{root / 'test.db'}"), + enabled_modules=(), + manifest_factories=(access_manifest,), + base_routers=(router,), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + app = create_app(config) + + database = get_database() + Base.metadata.create_all(bind=database.engine) + with database.session() as session: + tenant = Tenant(id=tenant_id, slug="capability", name="Capability Tenant", settings={}) + account = Account( + id=account_id, + email="capability@example.test", + normalized_email="capability@example.test", + display_name="Capability User", + ) + user = User( + id=user_id, + tenant_id=tenant_id, + account_id=account_id, + email=account.email, + display_name=account.display_name, + settings={}, + mail_profile_policy={}, + ) + session.add_all([tenant, account, user]) + session.commit() + + with TestClient(app) as client: + response = client.get("/api/v1/capability-principal") + + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual( + { + "account_id": account_id, + "tenant_id": tenant_id, + "user_id": user_id, + "has_demo_scope": True, + }, + response.json(), + ) + finally: + shutil.rmtree(root, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_database_migrations.py b/tests/test_database_migrations.py index 7b3570d..5970afc 100644 --- a/tests/test_database_migrations.py +++ b/tests/test_database_migrations.py @@ -6,6 +6,7 @@ from pathlib import Path from alembic import command from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory from sqlalchemy import create_engine, inspect, text from govoplan_core.db.base import Base @@ -18,6 +19,14 @@ from govoplan_core.db.migrations import ( ) +def configured_migration_heads(database_url: str) -> set[str]: + return set(ScriptDirectory.from_config(alembic_config(database_url=database_url)).get_heads()) + + +def database_migration_heads(connection) -> set[str]: + return set(MigrationContext.configure(connection).get_current_heads()) + + class DatabaseMigrationTests(unittest.TestCase): def test_repairs_create_all_schema_drift_and_upgrades_to_head(self) -> None: with tempfile.TemporaryDirectory(prefix="msm-migration-test-") as directory: @@ -52,7 +61,7 @@ class DatabaseMigrationTests(unittest.TestCase): engine = create_engine(url) try: with engine.connect() as connection: - current = MigrationContext.configure(connection).get_current_revision() + current = database_migration_heads(connection) inspector = inspect(connection) columns = { column["name"] @@ -93,7 +102,8 @@ class DatabaseMigrationTests(unittest.TestCase): WHERE role.slug = 'system_owner' AND account.is_active = 1 """ )).scalar_one() - self.assertEqual(current, result.current_revision) + self.assertEqual(current, configured_migration_heads(url)) + self.assertEqual(result.current_revision, ",".join(sorted(current))) self.assertIn("user_lock_state", columns) self.assertIn("user_locked_at", columns) self.assertIn("user_locked_by_user_id", columns) @@ -212,10 +222,9 @@ class DatabaseMigrationTests(unittest.TestCase): engine = create_engine(url) try: with engine.connect() as connection: - self.assertEqual( - MigrationContext.configure(connection).get_current_revision(), - result.current_revision, - ) + current = database_migration_heads(connection) + self.assertEqual(current, configured_migration_heads(url)) + self.assertEqual(result.current_revision, ",".join(sorted(current))) profile_columns = { column["name"] for column in inspect(connection).get_columns("mail_server_profiles") diff --git a/tests/test_module_system.py b/tests/test_module_system.py index 5688aeb..9b01a5f 100644 --- a/tests/test_module_system.py +++ b/tests/test_module_system.py @@ -1,7 +1,10 @@ from __future__ import annotations import importlib +import base64 +import json import os +import shlex import shutil import subprocess import sys @@ -9,8 +12,12 @@ import tempfile import textwrap import tomllib import unittest +from dataclasses import replace from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch + +from sqlalchemy import Column, Integer, MetaData, Table, inspect # Keep the default app import side effect from bootstrapping a development DB. _TEST_ROOT = Path(tempfile.mkdtemp(prefix="govoplan-module-tests-")) @@ -20,15 +27,68 @@ os.environ.setdefault("DEV_BOOTSTRAP_ENABLED", "false") os.environ.setdefault("CELERY_ENABLED", "false") from fastapi.testclient import TestClient +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from govoplan_core.core.migrations import migration_metadata_plan +from govoplan_core.core.module_management import ( + ModuleInstallPlan, + ModuleInstallPlanItem, + ModuleManagementError, + desired_modules_after_package_plan, + module_install_plan_commands, + normalize_module_install_plan_item, + saved_desired_enabled_modules, + saved_module_install_plan, + save_desired_enabled_modules, + save_module_install_plan, + plan_desired_enabled_modules, +) +from govoplan_core.core.maintenance import MAINTENANCE_ACCESS_SCOPE, MaintenanceMode, saved_maintenance_mode, save_maintenance_mode +from govoplan_core.core.module_installer import ( + cancel_module_installer_request, + claim_next_module_installer_request, + list_module_installer_runs, + list_module_installer_requests, + module_installer_daemon_status, + module_install_preflight, + module_installer_lock_status, + queue_module_installer_request, + read_module_installer_request, + read_module_installer_run, + retry_module_installer_request, + rollback_module_install_run, + run_module_install_plan, + supervise_module_install_plan, + update_module_installer_daemon_status, + update_module_installer_request, +) +from govoplan_core.core.module_license import module_license_decision, validate_module_license +from govoplan_core.core.module_package_catalog import ( + module_package_catalog, + record_module_package_catalog_acceptance, + sign_module_package_catalog, + validate_module_package_catalog, +) +from govoplan_core.core.modules import MigrationRetirementPlan, ModuleCompatibility, ModuleUninstallGuardResult +from govoplan_core.core.module_guards import drop_table_retirement_provider +from govoplan_core.core.modules import MigrationSpec, ModuleManifest +from govoplan_core.admin.models import SystemSettings +from govoplan_core.db.base import Base +from govoplan_core.db.session import configure_database, get_database +from govoplan_core.security.module_permissions import scopes_grant_compatible +from govoplan_core.security.permissions import scope_grants from govoplan_core.server.app import create_app from govoplan_core.server.config import GovoplanServerConfig from govoplan_core.server.registry import available_module_manifests, build_platform_registry from govoplan_files.backend.storage.backends import LocalFilesystemStorageBackend, StorageBackendError _MANIFEST_PROJECT_MODULES = { - "access": "govoplan_core", + "access": "govoplan_access", + "admin": "govoplan_admin", + "tenancy": "govoplan_tenancy", + "policy": "govoplan_policy", + "audit": "govoplan_audit", "files": "govoplan_files", "mail": "govoplan_mail", "campaigns": "govoplan_campaign", @@ -96,10 +156,51 @@ class ModuleSystemTests(unittest.TestCase): def test_discovers_installed_core_and_product_module_manifests(self) -> None: manifests = available_module_manifests() - self.assertTrue({"access", "files", "mail", "campaigns"}.issubset(manifests)) + self.assertTrue({"access", "admin", "tenancy", "policy", "audit", "files", "mail", "campaigns"}.issubset(manifests)) self.assertEqual(manifests["campaigns"].dependencies, ("access",)) self.assertEqual(manifests["campaigns"].optional_dependencies, ("files", "mail")) + def test_access_manifest_contributes_admin_frontend_metadata(self) -> None: + manifests = available_module_manifests() + access = manifests["access"] + + self.assertEqual(("/admin",), tuple(item.path for item in access.nav_items)) + self.assertIsNotNone(access.frontend) + assert access.frontend is not None + self.assertEqual("@govoplan/access-webui", access.frontend.package_name) + self.assertEqual(("/admin",), tuple(route.path for route in access.frontend.routes)) + self.assertEqual(("/admin",), tuple(item.path for item in access.frontend.nav_items)) + self.assertTrue(access.frontend.routes[0].required_any) + self.assertEqual("admin", access.frontend.nav_items[0].icon) + + def test_tenant_wildcard_grants_module_tenant_scopes(self) -> None: + self.assertTrue(scope_grants("tenant:*", "calendar:event:read")) + self.assertTrue(scopes_grant_compatible(["tenant:*"], "calendar:event:read")) + self.assertFalse(scope_grants("tenant:*", "system:settings:read")) + + def test_platform_modules_own_live_legacy_model_tables(self) -> None: + from govoplan_admin.backend.db.models import GovernanceTemplate + from govoplan_audit.backend.db.models import AuditLog + from govoplan_access.backend.db.base import AccessBase + from govoplan_access.backend.db.models import Account, ApiKey, AuthSession, User + from govoplan_core.admin.models import SystemSettings + from govoplan_core.db.base import Base + from govoplan_tenancy.backend.db.models import Tenant + + self.assertIs(AccessBase, Base) + self.assertEqual("accounts", Account.__tablename__) + self.assertEqual("tenants", Tenant.__tablename__) + self.assertEqual("users", User.__tablename__) + self.assertEqual("api_keys", ApiKey.__tablename__) + self.assertEqual("auth_sessions", AuthSession.__tablename__) + self.assertEqual("audit_log", AuditLog.__tablename__) + self.assertEqual("governance_templates", GovernanceTemplate.__tablename__) + self.assertEqual("system_settings", SystemSettings.__tablename__) + self.assertIn("accounts", Base.metadata.tables) + self.assertIn("tenants", Base.metadata.tables) + self.assertNotIn("access_accounts", Base.metadata.tables) + self.assertNotIn("access_auth_sessions", Base.metadata.tables) + def test_module_manifest_versions_match_source_project_versions(self) -> None: manifests = available_module_manifests() for manifest_id, package_module in _MANIFEST_PROJECT_MODULES.items(): @@ -109,13 +210,13 @@ class ModuleSystemTests(unittest.TestCase): def test_enabled_module_permutations_register_expected_routes(self) -> None: cases = ( - ("core_only", (), {"access"}, set()), - ("files_only", ("files",), {"access", "files"}, {"/api/v1/files"}), - ("mail_only", ("mail",), {"access", "mail"}, {"/api/v1/mail"}), - ("campaign_without_files_or_mail", ("campaigns",), {"access", "campaigns"}, {"/api/v1/campaigns"}), - ("campaign_without_files", ("campaigns", "mail"), {"access", "campaigns", "mail"}, {"/api/v1/campaigns", "/api/v1/mail"}), - ("campaign_with_files_but_no_mail", ("campaigns", "files"), {"access", "campaigns", "files"}, {"/api/v1/campaigns", "/api/v1/files"}), - ("full_product", ("campaigns", "files", "mail"), {"access", "campaigns", "files", "mail"}, {"/api/v1/campaigns", "/api/v1/files", "/api/v1/mail"}), + ("core_only", (), {"tenancy", "access"}, set()), + ("files_only", ("files",), {"tenancy", "access", "files"}, {"/api/v1/files"}), + ("mail_only", ("mail",), {"tenancy", "access", "mail"}, {"/api/v1/mail"}), + ("campaign_without_files_or_mail", ("campaigns",), {"tenancy", "access", "campaigns"}, {"/api/v1/campaigns"}), + ("campaign_without_files", ("campaigns", "mail"), {"tenancy", "access", "campaigns", "mail"}, {"/api/v1/campaigns", "/api/v1/mail"}), + ("campaign_with_files_but_no_mail", ("campaigns", "files"), {"tenancy", "access", "campaigns", "files"}, {"/api/v1/campaigns", "/api/v1/files"}), + ("full_product", ("campaigns", "files", "mail"), {"tenancy", "access", "campaigns", "files", "mail"}, {"/api/v1/campaigns", "/api/v1/files", "/api/v1/mail"}), ) for name, enabled_modules, expected_modules, expected_prefixes in cases: with self.subTest(name=name): @@ -127,15 +228,28 @@ class ModuleSystemTests(unittest.TestCase): self.assertEqual(prefix in expected_prefixes, has_prefix, f"{name}: {prefix}") with TestClient(app) as client: + health = client.get("/health") + self.assertEqual(health.status_code, 200, health.text) + self.assertEqual({"status": "ok"}, health.json()) response = client.get("/api/v1/platform/modules") self.assertEqual(response.status_code, 200, response.text) payload_modules = {item["id"] for item in response.json()["modules"]} self.assertEqual(expected_modules, payload_modules) + def test_governance_template_routes_are_contributed_by_admin_module(self) -> None: + access_only_app, _settings_obj = self._app_for_modules(()) + access_only_paths = {getattr(route, "path", "") for route in access_only_app.routes} + self.assertIn("/api/v1/admin/users", access_only_paths) + self.assertNotIn("/api/v1/admin/system/governance-templates", access_only_paths) + + admin_app, _settings_obj = self._app_for_modules(("admin",)) + admin_paths = {getattr(route, "path", "") for route in admin_app.routes} + self.assertIn("/api/v1/admin/system/governance-templates", admin_paths) + def _run_physical_absence_probe(self, *, enabled_modules: tuple[str, ...], blocked_modules: tuple[str, ...]) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-absence-probe-", dir=_TEST_ROOT)) - enabled = ",".join(("access", *enabled_modules)) if enabled_modules else "access" + enabled = ",".join(("tenancy", "access", *enabled_modules)) if enabled_modules else "tenancy,access" script = f""" import importlib.abc import json @@ -204,6 +318,1047 @@ print(json.dumps({{"modules": [item.id for item in registry.manifests()], "route self.assertEqual(files_enabled, registry.integration_enabled("campaigns", "files")) self.assertEqual(mail_enabled, registry.integration_enabled("campaigns", "mail")) + def test_module_management_plan_adds_protected_modules_and_required_dependencies(self) -> None: + manifests = available_module_manifests() + + plan = plan_desired_enabled_modules(("campaigns",), manifests) + + self.assertEqual(("tenancy", "access", "admin", "campaigns"), plan.enabled_modules) + self.assertEqual((), plan.added_dependencies) + + def test_module_install_plan_is_saved_alongside_desired_state(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-install-plan-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files @ git+ssh://git.example.invalid/add-ideas/govoplan-files.git@v0.1.4", + "webui_package": "@govoplan/files-webui", + "webui_ref": "git+ssh://git.example.invalid/add-ideas/govoplan-files.git#v0.1.4", + }]) + save_desired_enabled_modules(session, ("tenancy", "access", "files")) + session.commit() + + self.assertEqual(("files",), tuple(item.module_id for item in plan.items)) + with database.session() as session: + saved = saved_module_install_plan(session) + + self.assertEqual(("files",), tuple(item.module_id for item in saved.items)) + self.assertEqual(( + "python -m pip install 'govoplan-files @ git+ssh://git.example.invalid/add-ideas/govoplan-files.git@v0.1.4'", + "npm pkg set 'dependencies.@govoplan/files-webui=git+ssh://git.example.invalid/add-ideas/govoplan-files.git#v0.1.4' && npm install", + ), module_install_plan_commands(saved)) + + def test_module_install_plan_rejects_local_dependency_refs(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-install-plan-local-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + with self.assertRaises(ModuleManagementError): + save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "file:../govoplan-files", + }]) + + def test_module_install_plan_allows_uninstall_webui_package_without_ref(self) -> None: + item = normalize_module_install_plan_item({ + "module_id": "files", + "action": "uninstall", + "python_package": "govoplan-files", + "webui_package": "@govoplan/files-webui", + }) + plan = ModuleInstallPlan(items=(item,)) + + self.assertIsNone(item.webui_ref) + self.assertEqual(( + "python -m pip uninstall -y govoplan-files", + "npm pkg delete dependencies.@govoplan/files-webui && npm install", + ), module_install_plan_commands(plan)) + + def test_desired_modules_after_package_plan_applies_default_install_uninstall_state(self) -> None: + plan = ModuleInstallPlan(items=( + ModuleInstallPlanItem( + module_id="mail", + action="install", + python_package="govoplan-mail", + python_ref="govoplan-mail==0.1.4", + ), + ModuleInstallPlanItem( + module_id="files", + action="uninstall", + python_package="govoplan-files", + ), + )) + + self.assertEqual( + ("tenancy", "access", "mail"), + desired_modules_after_package_plan(("tenancy", "access", "files"), plan), + ) + + def test_maintenance_mode_round_trips_through_system_settings(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-maintenance-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + saved = save_maintenance_mode(session, MaintenanceMode(enabled=True, message="Tagged module install")) + session.commit() + + self.assertTrue(saved.enabled) + with database.session() as session: + mode = saved_maintenance_mode(session) + + self.assertTrue(mode.enabled) + self.assertEqual("Tagged module install", mode.message) + self.assertEqual("system:maintenance:access", MAINTENANCE_ACCESS_SCOPE) + + def test_module_installer_preflight_blocks_active_module_uninstall(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-guard-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "uninstall", + "python_package": "govoplan-files", + }]) + session.commit() + + preflight = module_install_preflight( + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access", "files"), + desired_enabled=("tenancy", "access", "files"), + maintenance_mode=True, + ) + + self.assertFalse(preflight.allowed) + self.assertIn("module_active", {issue.code for issue in preflight.issues}) + self.assertIn("module_desired", {issue.code for issue in preflight.issues}) + + def test_module_installer_preflight_blocks_incompatible_manifest(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-compat-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + + available = dict(available_module_manifests()) + available["files"] = replace( + available["files"], + compatibility=ModuleCompatibility(core_version_min="999.0.0"), + ) + + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + ) + + self.assertFalse(preflight.allowed) + self.assertIn("core_version_too_low", {issue.code for issue in preflight.issues}) + self.assertTrue(preflight.checklist) + + def test_module_installer_preflight_requires_python_package_for_install_rollback(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-install-rollback-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + + preflight = module_install_preflight( + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + ) + + self.assertFalse(preflight.allowed) + self.assertIn("python_package_required", {issue.code for issue in preflight.issues}) + + def test_module_installer_preflight_uses_module_uninstall_guards(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-module-guard-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "uninstall", + "python_package": "govoplan-files", + }]) + session.commit() + + available = dict(available_module_manifests()) + migration = available["files"].migration_spec + self.assertIsNotNone(migration) + assert migration is not None + available["files"] = replace( + available["files"], + migration_spec=replace(migration, retirement_supported=False, retirement_provider=None), + uninstall_guard_providers=(lambda _session, _module_id: ( + ModuleUninstallGuardResult("blocker", "stored_data_present", "Stored files remain."), + ),), + ) + + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + session=session, + ) + + self.assertFalse(preflight.allowed) + self.assertIn("stored_data_present", {issue.code for issue in preflight.issues}) + self.assertIn("migration_retirement_not_requested", {issue.code for issue in preflight.issues}) + + def test_module_installer_allows_non_destructive_uninstall_for_migration_owner(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-nondestructive-uninstall-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "uninstall", + "python_package": "govoplan-files", + }]) + session.commit() + + available = dict(available_module_manifests()) + migration = available["files"].migration_spec + self.assertIsNotNone(migration) + assert migration is not None + available["files"] = replace( + available["files"], + migration_spec=replace(migration, retirement_supported=False, retirement_provider=None), + uninstall_guard_providers=(), + ) + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + session=session, + ) + + self.assertTrue(preflight.allowed) + issues = {issue.code: issue for issue in preflight.issues} + self.assertEqual("warning", issues["migration_retirement_not_requested"].severity) + + def test_module_installer_preflight_uses_migration_retirement_provider(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-retirement-provider-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "uninstall", + "python_package": "govoplan-files", + }]) + session.commit() + + available = dict(available_module_manifests()) + migration = available["files"].migration_spec + self.assertIsNotNone(migration) + assert migration is not None + available["files"] = replace( + available["files"], + migration_spec=replace( + migration, + retirement_supported=True, + retirement_provider=lambda _session, _module_id: MigrationRetirementPlan( + supported=True, + summary="Retirement provider can preserve historical migration state.", + warnings=("Operator must verify archived file storage.",), + ), + ), + ) + + preflight = module_install_preflight( + plan=plan, + available=available, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + session=session, + ) + + codes = {issue.code for issue in preflight.issues} + self.assertTrue(preflight.allowed) + self.assertNotIn("migration_retirement_not_requested", codes) + self.assertIn("migration_retirement_supported", codes) + self.assertIn("migration_retirement_warning", codes) + + def test_module_installer_destroy_data_retirement_drops_module_tables(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-destroy-data-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + metadata = MetaData() + table = Table("retirement_example", metadata, Column("id", Integer, primary_key=True)) + metadata.create_all(bind=database.engine) + example_model = type("ExampleModel", (), {"__table__": table}) + manifest = ModuleManifest( + id="example", + name="Example", + version="0.1.0", + migration_spec=MigrationSpec( + module_id="example", + metadata=metadata, + retirement_supported=True, + retirement_provider=drop_table_retirement_provider(example_model, label="Example"), + ), + ) + + def fake_run(*_args, **_kwargs): + return SimpleNamespace(returncode=0, stdout="", stderr="") + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "example", + "action": "uninstall", + "python_package": "govoplan-example", + "destroy_data": True, + }]) + session.commit() + + preflight = module_install_preflight( + plan=plan, + available={"example": manifest}, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + maintenance_mode=True, + session=session, + ) + self.assertTrue(preflight.allowed) + self.assertIn("migration_retirement_destroy_planned", {issue.code for issue in preflight.issues}) + + with patch("govoplan_core.core.module_installer.subprocess.run", side_effect=fake_run): + result = run_module_install_plan( + session=session, + plan=plan, + available={"example": manifest}, + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=settings.database_url, + runtime_dir=root / "installer", + ) + + self.assertEqual("applied", result.status) + self.assertFalse(inspect(database.engine).has_table("retirement_example")) + record = json.loads(result.record_path.read_text(encoding="utf-8")) + self.assertEqual("example", record["retirements"][0]["module_id"]) + self.assertIn("database_backup", record["snapshot"]) + + def test_module_installer_dry_run_writes_run_record_without_applying(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-dry-run-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + + result = run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=settings.database_url, + runtime_dir=root / "installer", + migrate_database=True, + dry_run=True, + ) + + self.assertEqual("dry-run", result.status) + self.assertTrue(result.record_path.exists()) + record_text = result.record_path.read_text(encoding="utf-8") + self.assertIn("pip", record_text) + self.assertIn("database_backup", record_text) + self.assertIn("govoplan_core.commands.module_verify", record_text) + self.assertIn("govoplan_core.commands.init_db", record_text) + runs = list_module_installer_runs(runtime_dir=root / "installer") + self.assertEqual((result.run_id,), tuple(item["run_id"] for item in runs)) + self.assertFalse(module_installer_lock_status(runtime_dir=root / "installer")["locked"]) + record = read_module_installer_run(runtime_dir=root / "installer", run_id=result.run_id) + self.assertEqual(result.run_id, record["run_id"]) + + def test_supervised_module_install_rollback_restores_desired_modules(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-desired-rollback-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + + def fake_run(*_args, **kwargs): + if kwargs.get("shell"): + return SimpleNamespace(returncode=1, stdout="", stderr="restart failed") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "example", + "action": "install", + "python_package": "govoplan-example", + "python_ref": "govoplan-example==0.1.4", + }]) + save_desired_enabled_modules(session, ("tenancy", "access")) + session.commit() + + with patch("govoplan_core.core.module_installer.subprocess.run", side_effect=fake_run): + result = supervise_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url=settings.database_url, + runtime_dir=root / "installer", + restart_command="restart-fails", + ) + + restored_desired = saved_desired_enabled_modules(session, ("tenancy", "access")) + restored_plan = saved_module_install_plan(session) + + self.assertEqual("rolled-back", result.status) + self.assertEqual(("tenancy", "access"), restored_desired) + self.assertEqual(("planned",), tuple(item.status for item in restored_plan.items)) + + def test_module_installer_external_database_backup_command_is_recorded(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-external-backup-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + backup_script = ( + "import json, os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8'); " + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_METADATA']).write_text(json.dumps({'snapshot': 'external'}), encoding='utf-8')" + ) + backup_command = f"{sys.executable} -c {shlex.quote(backup_script)}" + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + + result = run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url="postgresql://db.example.invalid/govoplan", + runtime_dir=root / "installer", + migrate_database=True, + database_backup_command=backup_command, + database_restore_command="restore-database", + dry_run=True, + ) + + record = json.loads(result.record_path.read_text(encoding="utf-8")) + database_backup = record["snapshot"]["database_backup"] + self.assertEqual("external", database_backup["type"]) + self.assertEqual({"snapshot": "external"}, database_backup["metadata"]) + self.assertEqual("backup", (result.record_path.parent / "database.external.backup").read_text(encoding="utf-8")) + + def test_module_installer_external_database_restore_check_runs_before_migrations(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-restore-check-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + backup_script = ( + "import os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).write_text('backup', encoding='utf-8')" + ) + check_script = ( + "import os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore-check.marker').write_text(" + "pathlib.Path(os.environ['GOVOPLAN_DATABASE_BACKUP_PATH']).read_text(encoding='utf-8'), encoding='utf-8')" + ) + + with database.session() as session: + save_maintenance_mode(session, MaintenanceMode(enabled=True)) + plan = save_module_install_plan(session, [{ + "module_id": "files", + "action": "install", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }]) + session.commit() + + result = run_module_install_plan( + session=session, + plan=plan, + available=available_module_manifests(), + current_enabled=("tenancy", "access"), + desired_enabled=("tenancy", "access"), + database_url="postgresql://db.example.invalid/govoplan", + runtime_dir=root / "installer", + migrate_database=True, + database_backup_command=f"{sys.executable} -c {shlex.quote(backup_script)}", + database_restore_command="restore-database", + database_restore_check_command=f"{sys.executable} -c {shlex.quote(check_script)}", + dry_run=True, + ) + + record = json.loads(result.record_path.read_text(encoding="utf-8")) + database_backup = record["snapshot"]["database_backup"] + self.assertEqual(0, database_backup["restore_check"]["return_code"]) + self.assertEqual("backup", (result.record_path.parent / "restore-check.marker").read_text(encoding="utf-8")) + + def test_module_installer_external_database_restore_command_runs_on_rollback(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-external-restore-", dir=_TEST_ROOT)) + runtime_dir = root / "installer" + run_id = "external-restore-test" + run_dir = runtime_dir / "runs" / run_id + run_dir.mkdir(parents=True) + restore_script = ( + "import os, pathlib; " + "pathlib.Path(os.environ['GOVOPLAN_INSTALLER_RUN_DIR'], 'restore.marker').write_text(" + "os.environ.get('GOVOPLAN_DATABASE_URL', ''), encoding='utf-8')" + ) + restore_command = f"{sys.executable} -c {shlex.quote(restore_script)}" + (run_dir / "database.external.backup").write_text("backup", encoding="utf-8") + (run_dir / "record.json").write_text(json.dumps({ + "run_id": run_id, + "plan": [], + "snapshot": { + "database_backup": { + "type": "external", + "database_url": "postgresql://db.example.invalid/govoplan", + "backup_path": "database.external.backup", + "metadata_path": "database-backup-metadata.json", + "restore_command": restore_command, + }, + }, + }), encoding="utf-8") + + result = rollback_module_install_run(run_id=run_id, runtime_dir=runtime_dir, npm_bin="npm") + + self.assertEqual("rolled-back", result.status) + self.assertEqual("postgresql://db.example.invalid/govoplan", (run_dir / "restore.marker").read_text(encoding="utf-8")) + record = json.loads((run_dir / "record.json").read_text(encoding="utf-8")) + self.assertTrue(record["rollback_database_restore"]["restored"]) + + def test_module_installer_request_queue_round_trips(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-request-queue-", dir=_TEST_ROOT)) + runtime_dir = root / "installer" + + queued = queue_module_installer_request( + runtime_dir=runtime_dir, + requested_by="user-1", + options={"migrate_database": True, "health_urls": ["http://127.0.0.1:8000/health"]}, + ) + request_id = str(queued["request_id"]) + + self.assertEqual("queued", queued["status"]) + self.assertEqual((request_id,), tuple(item["request_id"] for item in list_module_installer_requests(runtime_dir=runtime_dir))) + claimed = claim_next_module_installer_request(runtime_dir=runtime_dir) + self.assertIsNotNone(claimed) + assert claimed is not None + self.assertEqual(request_id, claimed["request_id"]) + self.assertEqual("running", claimed["status"]) + updated = update_module_installer_request( + runtime_dir=runtime_dir, + request_id=request_id, + patch={"status": "completed", "result": {"return_code": 0}}, + ) + self.assertEqual("completed", updated["status"]) + self.assertEqual("completed", read_module_installer_request(runtime_dir=runtime_dir, request_id=request_id)["status"]) + + cancellable = queue_module_installer_request(runtime_dir=runtime_dir, requested_by="user-1") + cancelled = cancel_module_installer_request( + runtime_dir=runtime_dir, + request_id=str(cancellable["request_id"]), + cancelled_by="user-2", + ) + self.assertEqual("cancelled", cancelled["status"]) + self.assertEqual("user-2", cancelled["cancelled_by"]) + + retry = retry_module_installer_request( + runtime_dir=runtime_dir, + request_id=str(cancellable["request_id"]), + requested_by="user-3", + ) + self.assertEqual("queued", retry["status"]) + self.assertEqual(cancellable["request_id"], retry["retry_of"]) + + def test_module_installer_daemon_status_reports_heartbeat(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-daemon-status-", dir=_TEST_ROOT)) + runtime_dir = root / "installer" + + initial = module_installer_daemon_status(runtime_dir=runtime_dir) + self.assertFalse(initial["running"]) + + status = update_module_installer_daemon_status( + runtime_dir=runtime_dir, + patch={"status": "polling", "current_request_id": None}, + ) + + self.assertTrue(status["running"]) + self.assertEqual("polling", status["status"]) + + def test_module_package_catalog_loads_json_entries(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + catalog_path.write_text(json.dumps({ + "modules": [{ + "module_id": "files", + "name": "Files", + "version": "0.1.4", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + "tags": ["official"], + }], + }), encoding="utf-8") + + catalog = module_package_catalog(catalog_path) + + self.assertEqual("files", catalog[0]["module_id"]) + self.assertEqual("install", catalog[0]["action"]) + self.assertEqual(["official"], catalog[0]["tags"]) + + validation = validate_module_package_catalog(catalog_path) + self.assertTrue(validation["valid"]) + self.assertEqual("files", validation["modules"][0]["module_id"]) + + def test_module_package_catalog_validation_reports_bad_entries(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-invalid-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + catalog_path.write_text(json.dumps({"modules": [{"name": "Missing id"}]}), encoding="utf-8") + + validation = validate_module_package_catalog(catalog_path) + + self.assertFalse(validation["valid"]) + self.assertIn("module_id", str(validation["error"])) + + def test_module_package_catalog_validates_signed_approved_channel(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-signed-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + private_key_path = root / "catalog-private.pem" + signed_path = root / "catalog.signed.json" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + catalog_path.write_text(json.dumps({ + "channel": "stable", + "modules": [{ + "module_id": "files", + "python_package": "govoplan-files", + "python_ref": "govoplan-files==0.1.4", + }], + }), encoding="utf-8") + + sign_module_package_catalog( + path=catalog_path, + key_id="release-1", + private_key_path=private_key_path, + output_path=signed_path, + ) + validation = validate_module_package_catalog( + signed_path, + require_trusted=True, + approved_channels=("stable",), + trusted_keys={"release-1": base64.b64encode(public_key).decode("ascii")}, + ) + + self.assertTrue(validation["valid"], validation["error"]) + self.assertTrue(validation["signed"]) + self.assertTrue(validation["trusted"]) + self.assertEqual("stable", validation["channel"]) + + def test_module_package_catalog_rejects_unapproved_channel_when_required(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-channel-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + catalog_path.write_text(json.dumps({"channel": "nightly", "modules": [{"module_id": "files"}]}), encoding="utf-8") + + validation = validate_module_package_catalog(catalog_path, approved_channels=("stable",)) + + self.assertFalse(validation["valid"]) + self.assertIn("not approved", str(validation["error"])) + + def test_module_package_catalog_rejects_expired_catalog(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-expired-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + catalog_path.write_text(json.dumps({ + "channel": "stable", + "sequence": 1, + "generated_at": "2000-01-01T00:00:00Z", + "expires_at": "2000-01-02T00:00:00Z", + "modules": [{"module_id": "files"}], + }), encoding="utf-8") + + validation = validate_module_package_catalog(catalog_path) + + self.assertFalse(validation["valid"]) + self.assertIn("expired", str(validation["error"])) + + def test_module_package_catalog_records_sequence_acceptance(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-sequence-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + state_path = root / "sequence-state.json" + catalog_path.write_text(json.dumps({ + "channel": "stable", + "sequence": 7, + "generated_at": "2026-07-07T00:00:00Z", + "expires_at": "2030-12-31T23:59:59Z", + "modules": [{"module_id": "files"}], + }), encoding="utf-8") + + with patch.dict(os.environ, { + "GOVOPLAN_MODULE_PACKAGE_CATALOG_SEQUENCE_STATE": str(state_path), + "GOVOPLAN_MODULE_PACKAGE_CATALOG_ENFORCE_SEQUENCE": "true", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNELS": "", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_REQUIRE_SIGNATURE": "", + }): + validation = validate_module_package_catalog(catalog_path) + self.assertTrue(validation["valid"], validation["error"]) + record_module_package_catalog_acceptance(validation) + replay = validate_module_package_catalog(catalog_path) + + self.assertFalse(replay["valid"]) + self.assertIn("already accepted", str(replay["error"])) + + def test_module_license_decision_is_observe_only_unless_enforced(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-license-observe-", dir=_TEST_ROOT)) + missing_license = root / "missing-license.json" + + with patch.dict(os.environ, { + "GOVOPLAN_LICENSE_FILE": str(missing_license), + "GOVOPLAN_LICENSE_ENFORCEMENT": "", + }): + observe = module_license_decision(("module.mail",)) + + with patch.dict(os.environ, { + "GOVOPLAN_LICENSE_FILE": str(missing_license), + "GOVOPLAN_LICENSE_ENFORCEMENT": "true", + }): + enforced = module_license_decision(("module.mail",)) + + self.assertTrue(observe["allowed"]) + self.assertFalse(observe["enforced"]) + self.assertFalse(enforced["allowed"]) + self.assertTrue(enforced["enforced"]) + + def test_module_license_validates_signed_feature_license(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-license-signed-", dir=_TEST_ROOT)) + license_path = root / "license.json" + private_key = Ed25519PrivateKey.generate() + public_key = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + payload = { + "license_id": "test-license", + "subject": "Test installation", + "features": ["module.mail"], + "valid_from": "2026-01-01T00:00:00Z", + "valid_until": "2030-12-31T23:59:59Z", + } + signature = private_key.sign(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")) + payload["signature"] = { + "algorithm": "ed25519", + "key_id": "license-1", + "value": base64.b64encode(signature).decode("ascii"), + } + license_path.write_text(json.dumps(payload), encoding="utf-8") + trusted_keys = {"license-1": base64.b64encode(public_key).decode("ascii")} + + validation = validate_module_license(license_path, trusted_keys=trusted_keys, require_trusted=True) + + self.assertTrue(validation["valid"], validation["error"]) + self.assertTrue(validation["trusted"]) + self.assertEqual(["module.mail"], validation["features"]) + + def test_module_installer_validates_package_catalog_from_cli(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-package-catalog-cli-", dir=_TEST_ROOT)) + catalog_path = root / "catalog.json" + catalog_path.write_text(json.dumps({"modules": [{"module_id": "files"}]}), encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + "-m", + "govoplan_core.commands.module_installer", + "--validate-package-catalog", + str(catalog_path), + "--format", + "json", + ], + cwd=str(Path(__file__).resolve().parents[1]), + env=os.environ.copy(), + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertTrue(json.loads(result.stdout)["valid"]) + + def test_module_installer_daemon_once_exits_with_empty_queue(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-daemon-once-", dir=_TEST_ROOT)) + result = subprocess.run( + [ + sys.executable, + "-m", + "govoplan_core.commands.module_installer", + "--daemon-once", + "--runtime-dir", + str(root / "installer"), + "--database-url", + f"sqlite:///{root / 'daemon.db'}", + ], + cwd=str(Path(__file__).resolve().parents[1]), + env=os.environ.copy(), + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(0, result.returncode, result.stderr) + + def test_module_installer_rollback_rebuilds_webui_when_original_run_built(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-installer-rollback-webui-", dir=_TEST_ROOT)) + runtime_dir = root / "installer" + run_id = "rollback-webui-test" + run_dir = runtime_dir / "runs" / run_id + webui_root = root / "webui" + run_dir.mkdir(parents=True) + webui_root.mkdir() + (webui_root / "package.json").write_text('{"scripts":{"build":"vite build"}}', encoding="utf-8") + (run_dir / "package.json.before").write_text('{"scripts":{"build":"vite build"},"dependencies":{}}', encoding="utf-8") + (run_dir / "package-lock.json.before").write_text('{"lockfileVersion":3}', encoding="utf-8") + (run_dir / "record.json").write_text(json.dumps({ + "run_id": run_id, + "build_webui": True, + "plan": [{ + "module_id": "example", + "action": "install", + "python_package": "govoplan-example", + }], + "snapshot": { + "webui_root": str(webui_root), + "package_json": "package.json.before", + "package_lock": "package-lock.json.before", + }, + }), encoding="utf-8") + + with patch("govoplan_core.core.module_installer.subprocess.run") as run_mock: + run_mock.return_value = SimpleNamespace(returncode=0, stdout="", stderr="") + result = rollback_module_install_run(run_id=run_id, runtime_dir=runtime_dir, npm_bin="npm") + + commands = [call.args[0] for call in run_mock.call_args_list] + self.assertEqual("rolled-back", result.status) + self.assertIn([sys.executable, "-m", "pip", "uninstall", "-y", "govoplan-example"], commands) + self.assertIn(["npm", "install"], commands) + self.assertIn(["npm", "run", "build"], commands) + + def test_create_app_uses_saved_desired_module_state_on_startup(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-state-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + with database.session() as session: + session.add(SystemSettings( + id="global", + settings={"module_management": {"desired_enabled": ["files"]}}, + )) + session.commit() + + config = GovoplanServerConfig( + title="GovOPlaN Module State Test", + version="test", + settings=settings, + enabled_modules=(), + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + + app = create_app(config) + with TestClient(app) as client: + response = client.get("/api/v1/platform/modules") + + self.assertEqual(response.status_code, 200, response.text) + payload_modules = {item["id"] for item in response.json()["modules"]} + self.assertEqual({"tenancy", "access", "files"}, payload_modules) + + def test_create_app_ignores_saved_desired_modules_that_are_not_installed(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-state-stale-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + with database.session() as session: + session.add(SystemSettings( + id="global", + settings={"module_management": {"desired_enabled": ["files", "missing-module"]}}, + )) + session.commit() + + config = GovoplanServerConfig( + title="GovOPlaN Module State Test", + version="test", + settings=settings, + enabled_modules=(), + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + + app = create_app(config) + with TestClient(app) as client: + response = client.get("/api/v1/platform/modules") + + self.assertEqual(response.status_code, 200, response.text) + payload_modules = {item["id"] for item in response.json()["modules"]} + self.assertEqual({"tenancy", "access", "files"}, payload_modules) + + def test_create_app_preserves_admin_when_configured_and_saved_state_is_older(self) -> None: + root = Path(tempfile.mkdtemp(prefix="govoplan-module-state-admin-", dir=_TEST_ROOT)) + settings = _settings(root) + configure_database(settings.database_url) + database = get_database() + Base.metadata.create_all(bind=database.engine, tables=[SystemSettings.__table__]) + with database.session() as session: + session.add(SystemSettings( + id="global", + settings={"module_management": {"desired_enabled": ["files"]}}, + )) + session.commit() + + config = GovoplanServerConfig( + title="GovOPlaN Module State Test", + version="test", + settings=settings, + enabled_modules=("admin",), + base_routers=(), + post_module_routers=(), + extra_routers=(), + lifespan=None, + app_configurators=(), + ) + + app = create_app(config) + with TestClient(app) as client: + response = client.get("/api/v1/platform/modules") + + self.assertEqual(response.status_code, 200, response.text) + payload_modules = {item["id"] for item in response.json()["modules"]} + self.assertEqual({"tenancy", "access", "admin", "files"}, payload_modules) + + def test_module_lifecycle_enables_and_disables_routes_without_restart(self) -> None: + app, _settings_obj = self._app_for_modules(()) + lifecycle = getattr(app.state, "govoplan_lifecycle", None) + self.assertIsNotNone(lifecycle) + + with TestClient(app) as client: + response = client.get("/api/v1/platform/modules") + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual({"tenancy", "access"}, {item["id"] for item in response.json()["modules"]}) + self.assertFalse(any(getattr(route, "path", "") == "/api/v1/files" for route in app.routes)) + + result = lifecycle.apply_enabled_modules(("files",), migrate=False) + self.assertEqual(("files",), result.activated_modules) + response = client.get("/api/v1/platform/modules") + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual({"tenancy", "access", "files"}, {item["id"] for item in response.json()["modules"]}) + self.assertTrue(any(getattr(route, "path", "") == "/api/v1/files" for route in app.routes)) + self.assertEqual(401, client.get("/api/v1/files").status_code) + + result = lifecycle.apply_enabled_modules((), migrate=False) + self.assertEqual(("files",), result.deactivated_modules) + response = client.get("/api/v1/platform/modules") + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual({"tenancy", "access"}, {item["id"] for item in response.json()["modules"]}) + disabled_response = client.get("/api/v1/files") + self.assertEqual(404, disabled_response.status_code) + self.assertEqual("Module is disabled: files", disabled_response.json()["detail"]) + def test_module_permutations_start_when_absent_modules_are_physically_unavailable(self) -> None: cases = ( diff --git a/webui/package-lock.json b/webui/package-lock.json index d9edd22..0c35e35 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -1,13 +1,16 @@ { "name": "@govoplan/core-webui", - "version": "0.1.4", + "version": "0.1.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govoplan/core-webui", - "version": "0.1.4", + "version": "0.1.6", "dependencies": { + "@govoplan/access-webui": "file:../../govoplan-access/webui", + "@govoplan/admin-webui": "file:../../govoplan-admin/webui", + "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", "@govoplan/files-webui": "file:../../govoplan-files/webui", "@govoplan/mail-webui": "file:../../govoplan-mail/webui" @@ -31,9 +34,66 @@ "react-router-dom": "^7.1.1" } }, + "../../govoplan-access/webui": { + "name": "@govoplan/access-webui", + "version": "0.1.6", + "devDependencies": { + "typescript": "^5.7.2" + }, + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "lucide-react": "^0.555.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "../../govoplan-admin/webui": { + "name": "@govoplan/admin-webui", + "version": "0.1.6", + "devDependencies": { + "typescript": "^5.7.2" + }, + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "lucide-react": "^0.555.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "../../govoplan-calendar/webui": { + "name": "@govoplan/calendar-webui", + "version": "0.1.6", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "@vitejs/plugin-react": "^4.3.4", + "lucide-react": "^0.555.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1", + "typescript": "^5.7.2", + "vite": "^6.0.6" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-campaign/webui": { "name": "@govoplan/campaign-webui", - "version": "0.1.4", + "version": "0.1.6", "dependencies": { "read-excel-file": "^9.2.0" }, @@ -41,7 +101,7 @@ "typescript": "^5.7.2" }, "peerDependencies": { - "@govoplan/core-webui": "^0.1.4", + "@govoplan/core-webui": "^0.1.6", "lucide-react": "^0.555.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -55,9 +115,9 @@ }, "../../govoplan-files/webui": { "name": "@govoplan/files-webui", - "version": "0.1.4", + "version": "0.1.6", "peerDependencies": { - "@govoplan/core-webui": "^0.1.4", + "@govoplan/core-webui": "^0.1.6", "@vitejs/plugin-react": "^4.3.4", "lucide-react": "^0.555.0", "react": "^19.0.0", @@ -74,12 +134,12 @@ }, "../../govoplan-mail/webui": { "name": "@govoplan/mail-webui", - "version": "0.1.4", + "version": "0.1.6", "devDependencies": { "typescript": "^5.7.2" }, "peerDependencies": { - "@govoplan/core-webui": "^0.1.4", + "@govoplan/core-webui": "^0.1.6", "lucide-react": "^0.555.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -815,6 +875,18 @@ "node": ">=18" } }, + "node_modules/@govoplan/access-webui": { + "resolved": "../../govoplan-access/webui", + "link": true + }, + "node_modules/@govoplan/admin-webui": { + "resolved": "../../govoplan-admin/webui", + "link": true + }, + "node_modules/@govoplan/calendar-webui": { + "resolved": "../../govoplan-calendar/webui", + "link": true + }, "node_modules/@govoplan/campaign-webui": { "resolved": "../../govoplan-campaign/webui", "link": true diff --git a/webui/package-lock.release.json b/webui/package-lock.release.json index 1a24f12..25f21ca 100644 --- a/webui/package-lock.release.json +++ b/webui/package-lock.release.json @@ -1,16 +1,19 @@ { "name": "@govoplan/core-webui", - "version": "0.1.4", + "version": "0.1.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govoplan/core-webui", - "version": "0.1.4", + "version": "0.1.6", "dependencies": { - "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.4", - "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4", - "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4" + "@govoplan/access-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git#v0.1.6", + "@govoplan/admin-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git#v0.1.6", + "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git#v0.1.6", + "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.6", + "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.6", + "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.6" }, "devDependencies": { "@types/react": "^19.0.2", @@ -709,14 +712,65 @@ "node": ">=18" } }, + "node_modules/@govoplan/access-webui": { + "version": "0.1.6", + "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git#37828fe34099c17267e63cfbc3a78ae1075b6b52", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "lucide-react": "^0.555.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "node_modules/@govoplan/admin-webui": { + "version": "0.1.6", + "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git#0f2a9beca76c695a70a709bce2d843e6b1349277", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "lucide-react": "^0.555.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "node_modules/@govoplan/calendar-webui": { + "version": "0.1.6", + "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git#c3c867391c39813593930f0272a1a0142c70872b", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.6", + "@vitejs/plugin-react": "^4.3.4", + "lucide-react": "^0.555.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "react-router-dom": "^7.1.1", + "typescript": "^5.7.2", + "vite": "^6.0.6" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "node_modules/@govoplan/campaign-webui": { - "version": "0.1.4", - "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#fb6fb67d451858cca891d8a2fe5b6b8790f75f0d", + "version": "0.1.6", + "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#57f6066bf72ba817ce5a1973405d2422ae59678d", "dependencies": { "read-excel-file": "^9.2.0" }, "peerDependencies": { - "@govoplan/core-webui": "^0.1.4", + "@govoplan/core-webui": "^0.1.6", "lucide-react": "^0.555.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -729,10 +783,10 @@ } }, "node_modules/@govoplan/files-webui": { - "version": "0.1.4", - "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#ee5b06b1e7d60f458ceb9ed943369c55201698ce", + "version": "0.1.6", + "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#18e6c3eb9b776bd58c18a750725d6cd8a403bc5c", "peerDependencies": { - "@govoplan/core-webui": "^0.1.4", + "@govoplan/core-webui": "^0.1.6", "@vitejs/plugin-react": "^4.3.4", "lucide-react": "^0.555.0", "react": "^19.0.0", @@ -748,10 +802,10 @@ } }, "node_modules/@govoplan/mail-webui": { - "version": "0.1.4", - "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#3743b236130d6684b5e8d406f5070e3ff9e33d07", + "version": "0.1.6", + "resolved": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#b4b0c76455e5fddadd03af3b023c82678dedf36f", "peerDependencies": { - "@govoplan/core-webui": "^0.1.4", + "@govoplan/core-webui": "^0.1.6", "lucide-react": "^0.555.0", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/webui/package.json b/webui/package.json index 2077791..3381597 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/core-webui", - "version": "0.1.4", + "version": "0.1.6", "private": true, "type": "module", "main": "src/index.ts", @@ -25,6 +25,9 @@ "test:mail-components": "rm -rf .component-test-build && mkdir -p .component-test-build && printf '{\"type\":\"commonjs\"}\\n' > .component-test-build/package.json && tsc -p tsconfig.component-tests.json && node .component-test-build/tests/mail-components.test.js" }, "dependencies": { + "@govoplan/access-webui": "file:../../govoplan-access/webui", + "@govoplan/admin-webui": "file:../../govoplan-admin/webui", + "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", "@govoplan/files-webui": "file:../../govoplan-files/webui", "@govoplan/mail-webui": "file:../../govoplan-mail/webui" diff --git a/webui/package.release.json b/webui/package.release.json index 51d7f14..ada5f24 100644 --- a/webui/package.release.json +++ b/webui/package.release.json @@ -1,6 +1,6 @@ { "name": "@govoplan/core-webui", - "version": "0.1.4", + "version": "0.1.6", "private": true, "type": "module", "main": "src/index.ts", @@ -22,9 +22,12 @@ "preview": "vite preview --host 127.0.0.1 --port 4173" }, "dependencies": { - "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.4", - "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.4", - "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.4" + "@govoplan/access-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-access.git#v0.1.6", + "@govoplan/admin-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-admin.git#v0.1.6", + "@govoplan/calendar-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-calendar.git#v0.1.6", + "@govoplan/files-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-files.git#v0.1.6", + "@govoplan/mail-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-mail.git#v0.1.6", + "@govoplan/campaign-webui": "git+ssh://git@git.add-ideas.de/add-ideas/govoplan-campaign.git#v0.1.6" }, "devDependencies": { "lucide-react": "^0.555.0", diff --git a/webui/scripts/test-module-permutations.mjs b/webui/scripts/test-module-permutations.mjs index 79206aa..f37e226 100644 --- a/webui/scripts/test-module-permutations.mjs +++ b/webui/scripts/test-module-permutations.mjs @@ -1,6 +1,8 @@ import { spawnSync } from "node:child_process"; const packageByModule = { + access: "@govoplan/access-webui", + admin: "@govoplan/admin-webui", campaigns: "@govoplan/campaign-webui", files: "@govoplan/files-webui", mail: "@govoplan/mail-webui" @@ -8,12 +10,15 @@ const packageByModule = { const cases = [ { name: "core-only", modules: [] }, + { name: "access-only", modules: ["access"] }, + { name: "admin-only", modules: ["admin"] }, + { name: "access-with-admin", modules: ["access", "admin"] }, { name: "files-only", modules: ["files"] }, { name: "mail-only", modules: ["mail"] }, { name: "campaign-only", modules: ["campaigns"] }, { name: "campaign-with-files-no-mail", modules: ["campaigns", "files"] }, { name: "campaign-with-mail-no-files", modules: ["campaigns", "mail"] }, - { name: "full-product", modules: ["campaigns", "files", "mail"] } + { name: "full-product", modules: ["access", "admin", "campaigns", "files", "mail"] } ]; const npmExec = process.env.npm_execpath; diff --git a/webui/src/App.tsx b/webui/src/App.tsx index b6c60d0..480447f 100644 --- a/webui/src/App.tsx +++ b/webui/src/App.tsx @@ -1,30 +1,32 @@ import { Navigate, Route, Routes } from "react-router-dom"; import { lazy, Suspense, useEffect, useMemo, useState } from "react"; import { fetchMe } from "./api/auth"; -import { fetchPlatformModules } from "./api/platform"; +import { fetchPlatformModules, fetchPlatformStatus } from "./api/platform"; import { AUTH_REQUIRED_EVENT, loadApiSettings, saveApiSettings, type AuthRequiredEventDetail } from "./api/client"; -import type { ApiSettings, AuthInfo, LoginResponse, PlatformModuleInfo } from "./types"; +import type { ApiSettings, AuthInfo, LoginResponse, PlatformModuleInfo, PlatformWebModule } from "./types"; import AppShell from "./layout/AppShell"; import PublicLandingPage from "./features/auth/PublicLandingPage"; import LoginModal from "./features/auth/LoginModal"; import { PermissionBoundary } from "./components/AccessBoundary"; -import { adminReadScopes } from "./utils/permissions"; -import { firstAccessibleRoute, navItemsForModules, resolveInstalledWebModules, routeContributionsForModules } from "./platform/modules"; +import { firstAccessibleRoute, loadRemoteWebModules, navItemsForModules, resolveInstalledWebModules, routeContributionsForModules } from "./platform/modules"; import { PlatformModulesProvider } from "./platform/ModuleContext"; +import { PLATFORM_MODULES_CHANGED_EVENT } from "./platform/moduleEvents"; import { UnsavedChangesProvider } from "./components/UnsavedChangesGuard"; const DashboardPage = lazy(() => import("./features/dashboard/DashboardPage")); const SettingsPage = lazy(() => import("./features/settings/SettingsPage")); -const AdminPage = lazy(() => import("./features/admin/AdminPage")); export default function App() { const [settings, setSettings] = useState(() => loadApiSettings()); const [auth, setAuth] = useState(null); const [checkingSession, setCheckingSession] = useState(true); const [platformModules, setPlatformModules] = useState(null); + const [remoteWebModules, setRemoteWebModules] = useState([]); + const [maintenanceMode, setMaintenanceMode] = useState<{ enabled: boolean; message?: string | null }>({ enabled: false, message: null }); const [reloginMessage, setReloginMessage] = useState(""); - const webModules = useMemo(() => resolveInstalledWebModules(platformModules), [platformModules]); + const localWebModules = useMemo(() => resolveInstalledWebModules(platformModules), [platformModules]); + const webModules = useMemo(() => mergeWebModules(localWebModules, remoteWebModules), [localWebModules, remoteWebModules]); const navItems = useMemo(() => navItemsForModules(webModules), [webModules]); const moduleRoutes = useMemo(() => routeContributionsForModules(webModules), [webModules]); @@ -75,6 +77,18 @@ export default function App() { return () => window.removeEventListener(AUTH_REQUIRED_EVENT, handleAuthRequired); }, [auth]); + useEffect(() => { + let cancelled = false; + fetchPlatformStatus(settings) + .then((response) => { + if (!cancelled) setMaintenanceMode(response.maintenance_mode); + }) + .catch(() => { + if (!cancelled) setMaintenanceMode({ enabled: false, message: null }); + }); + return () => { cancelled = true; }; + }, [settings.apiBaseUrl]); + useEffect(() => { let cancelled = false; setCheckingSession(true); @@ -87,6 +101,7 @@ export default function App() { saveApiSettings(cleared); setAuth(null); setPlatformModules(null); + setRemoteWebModules([]); } }) .finally(() => { @@ -99,13 +114,52 @@ export default function App() { if (!auth) return; let cancelled = false; - fetchPlatformModules(settings) - .then((response) => { if (!cancelled) setPlatformModules(response.modules); }) - .catch(() => { if (!cancelled) setPlatformModules(null); }); + let inFlight = false; + let lastRefreshAt = 0; + function loadModules() { + inFlight = true; + lastRefreshAt = Date.now(); + return fetchPlatformModules(settings) + .then((response) => { if (!cancelled) setPlatformModules(response.modules); }) + .catch(() => { if (!cancelled) setPlatformModules(null); }) + .finally(() => { inFlight = false; }); + } - return () => { cancelled = true; }; + void loadModules(); + + function handleModulesChanged() { + void loadModules(); + } + + function refreshVisibleModules() { + if (document.visibilityState === "hidden" || inFlight) return; + if (Date.now() - lastRefreshAt < 5_000) return; + void loadModules(); + } + + window.addEventListener(PLATFORM_MODULES_CHANGED_EVENT, handleModulesChanged); + window.addEventListener("focus", refreshVisibleModules); + document.addEventListener("visibilitychange", refreshVisibleModules); + return () => { + cancelled = true; + window.removeEventListener(PLATFORM_MODULES_CHANGED_EVENT, handleModulesChanged); + window.removeEventListener("focus", refreshVisibleModules); + document.removeEventListener("visibilitychange", refreshVisibleModules); + }; }, [auth?.user.id, auth?.active_tenant?.id, auth?.tenant.id, settings.apiBaseUrl, settings.apiKey]); + useEffect(() => { + let cancelled = false; + if (!auth || !platformModules?.length) { + setRemoteWebModules([]); + return () => { cancelled = true; }; + } + loadRemoteWebModules(platformModules, localWebModules) + .then((modules) => { if (!cancelled) setRemoteWebModules(modules); }) + .catch(() => { if (!cancelled) setRemoteWebModules([]); }); + return () => { cancelled = true; }; + }, [auth?.user.id, auth?.active_tenant?.id, auth?.tenant.id, platformModules, localWebModules]); + useEffect(() => { if (!auth) return; @@ -159,7 +213,7 @@ export default function App() { - + @@ -182,17 +236,12 @@ export default function App() { path={route.path} element={ - {route.render({ settings, auth })} + {route.render({ settings, auth, onAuthChange: updateAuth })} } /> ))} } /> - - - - } /> } /> @@ -210,3 +259,16 @@ export default function App() { ); } + +function mergeWebModules(localModules: PlatformWebModule[], remoteModules: PlatformWebModule[]): PlatformWebModule[] { + if (remoteModules.length === 0) return localModules; + const seen = new Set(localModules.map((module) => module.id)); + return [ + ...localModules, + ...remoteModules.filter((module) => { + if (seen.has(module.id)) return false; + seen.add(module.id); + return true; + }) + ]; +} diff --git a/webui/src/api/platform.ts b/webui/src/api/platform.ts index 2a3de55..0c8336a 100644 --- a/webui/src/api/platform.ts +++ b/webui/src/api/platform.ts @@ -4,6 +4,13 @@ import { apiFetch } from "./client"; export type PlatformModulesResponse = { modules: PlatformModuleInfo[] }; +export type PlatformStatusResponse = { + maintenance_mode: { + enabled: boolean; + message?: string | null; + }; +}; + export type PlatformPermission = { scope: string; module_id: string; @@ -22,6 +29,10 @@ export async function fetchPlatformModules(settings: ApiSettings): Promise(settings, "/api/v1/platform/modules"); } +export async function fetchPlatformStatus(settings: ApiSettings): Promise { + return apiFetch(settings, "/api/v1/platform/status"); +} + export async function fetchPlatformPermissions(settings: ApiSettings): Promise { return apiFetch(settings, "/api/v1/platform/permissions"); } diff --git a/webui/src/features/admin/components/AdminIconButton.tsx b/webui/src/components/admin/AdminIconButton.tsx similarity index 91% rename from webui/src/features/admin/components/AdminIconButton.tsx rename to webui/src/components/admin/AdminIconButton.tsx index 914d701..8d7b59f 100644 --- a/webui/src/features/admin/components/AdminIconButton.tsx +++ b/webui/src/components/admin/AdminIconButton.tsx @@ -1,5 +1,5 @@ import type { ReactNode } from "react"; -import Button from "../../../components/Button"; +import Button from "../Button"; type Props = { label: string; diff --git a/webui/src/features/admin/components/AdminPageLayout.tsx b/webui/src/components/admin/AdminPageLayout.tsx similarity index 82% rename from webui/src/features/admin/components/AdminPageLayout.tsx rename to webui/src/components/admin/AdminPageLayout.tsx index 56535d6..e99ebd1 100644 --- a/webui/src/features/admin/components/AdminPageLayout.tsx +++ b/webui/src/components/admin/AdminPageLayout.tsx @@ -1,7 +1,7 @@ import type { ReactNode } from "react"; -import PageTitle from "../../../components/PageTitle"; -import LoadingFrame from "../../../components/LoadingFrame"; -import DismissibleAlert from "../../../components/DismissibleAlert"; +import DismissibleAlert from "../DismissibleAlert"; +import LoadingFrame from "../LoadingFrame"; +import PageTitle from "../PageTitle"; type Props = { title: string; @@ -19,7 +19,7 @@ export default function AdminPageLayout({ title, description, loading = false, - loadingLabel = "Loading administration data…", + loadingLabel = "Loading administration data...", error = "", success = "", actions, diff --git a/webui/src/features/admin/components/AdminSelectionList.tsx b/webui/src/components/admin/AdminSelectionList.tsx similarity index 100% rename from webui/src/features/admin/components/AdminSelectionList.tsx rename to webui/src/components/admin/AdminSelectionList.tsx diff --git a/webui/src/features/admin/adminUtils.ts b/webui/src/components/admin/adminUtils.ts similarity index 93% rename from webui/src/features/admin/adminUtils.ts rename to webui/src/components/admin/adminUtils.ts index 43568c1..e321026 100644 --- a/webui/src/features/admin/adminUtils.ts +++ b/webui/src/components/admin/adminUtils.ts @@ -18,7 +18,7 @@ export function adminErrorMessage(error: unknown): string { return message; } -export function formatDateTime(value?: string | null): string { +export function formatAdminDateTime(value?: string | null): string { return formatPlatformDateTime(value, { fallback: "Never" }); } diff --git a/webui/src/features/admin/AdminAuditPanel.tsx b/webui/src/features/admin/AdminAuditPanel.tsx deleted file mode 100644 index 2775d43..0000000 --- a/webui/src/features/admin/AdminAuditPanel.tsx +++ /dev/null @@ -1,123 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; -import { Search } from "lucide-react"; -import type { ApiSettings, AuthInfo } from "../../types"; -import { fetchAdminAudit, type AuditAdminItem } from "../../api/admin"; -import Button from "../../components/Button"; -import DataGrid, { type DataGridColumn, type DataGridQueryState } from "../../components/table/DataGrid"; -import Dialog from "../../components/Dialog"; -import AdminIconButton from "./components/AdminIconButton"; -import AdminPageLayout from "./components/AdminPageLayout"; -import { adminErrorMessage, formatDateTime } from "./adminUtils"; - -const DEFAULT_QUERY: DataGridQueryState = { - sort: { columnId: "time", direction: "desc" }, - filters: {} -}; - -export default function AdminAuditPanel({ - settings, - auth, - systemMode = false -}: { - settings: ApiSettings; - auth: AuthInfo; - systemMode?: boolean; -}) { - const [items, setItems] = useState([]); - const [total, setTotal] = useState(0); - const [page, setPage] = useState(1); - const [pageSize, setPageSize] = useState(10); - const [query, setQuery] = useState(DEFAULT_QUERY); - const [selected, setSelected] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(""); - const [reloadToken, setReloadToken] = useState(0); - const tenantId = (auth.active_tenant ?? auth.tenant).id; - - const load = useCallback(async () => { - setLoading(true); - setError(""); - try { - const sortColumn = query.sort?.columnId; - const response = await fetchAdminAudit(settings, { - scope: systemMode ? "system" : "tenant", - page, - pageSize, - sortBy: sortColumn && ["time", "actor", "action", "object", "tenant"].includes(sortColumn) - ? sortColumn as "time" | "actor" | "action" | "object" | "tenant" - : "time", - sortDirection: query.sort?.direction ?? "desc", - filters: query.filters - }); - setItems(response.items); - setTotal(response.total); - if (response.page !== page) setPage(response.page); - } catch (err) { - setError(adminErrorMessage(err)); - } finally { - setLoading(false); - } - }, [settings.accessToken, settings.apiBaseUrl, systemMode, tenantId, page, pageSize, query, reloadToken]); - - useEffect(() => { void load(); }, [load]); - - const handleQueryChange = useCallback((next: DataGridQueryState) => { - setQuery((current) => { - if (JSON.stringify(current) === JSON.stringify(next)) return current; - setPage(1); - return next; - }); - }, []); - - const columns = useMemo[]>(() => [ - { id: "time", header: "Time", width: 190, minWidth: 150, maxWidth: 260, resizable: true, sticky: "start", sortable: true, filterable: true, filterType: "date", value: (row) => row.created_at, render: (row) => formatDateTime(row.created_at) }, - { id: "actor", header: "Actor", width: 220, minWidth: 170, maxWidth: 360, resizable: true, sortable: true, filterable: true, value: (row) => row.actor_email || "System" }, - { id: "action", header: "Action", width: 250, minWidth: 170, maxWidth: 420, resizable: true, sortable: true, filterable: true, value: (row) => row.action }, - { id: "object", header: "Object", width: 300, minWidth: 180, maxWidth: 640, resizable: true, fill: true, sortable: true, filterable: true, value: (row) => `${row.object_type || "—"} ${row.object_id || ""}`.trim() }, - ...(systemMode ? [{ id: "tenant", header: "Tenant context", width: 190, minWidth: 150, maxWidth: 300, resizable: true, sortable: true, filterable: true, value: (row: AuditAdminItem) => row.tenant_id || "—" }] : []), - { id: "actions", header: "Actions", width: 70, sticky: "end", resizable: false, align: "right", render: (row) =>
} onClick={() => setSelected(row)} />
} - ], [systemMode]); - - const firstShown = total === 0 ? 0 : (page - 1) * pageSize + 1; - const lastShown = Math.min(total, page * pageSize); - - return ( - <> - setReloadToken((value) => value + 1)} disabled={loading}>Reload} - > -
- row.id} - emptyText="No administrative audit records found." - className="admin-audit-grid" - initialSort={{ columnId: "time", direction: "desc" }} - pagination={{ - mode: "server", - page, - pageSize, - totalRows: total, - pageSizeOptions: [10, 25, 50, 100, 250], - disabled: loading, - onPageChange: setPage, - onPageSizeChange: (next) => { setPageSize(next); setPage(1); } - }} - onQueryChange={handleQueryChange} - /> -
- - setSelected(null)} className="admin-dialog admin-dialog-wide" footer={}> - {selected && <>
Scope
{selected.scope}
Action
{selected.action}
Actor
{selected.actor_email || "System"}
Object
{selected.object_type || "—"} {selected.object_id || ""}
Tenant context
{selected.tenant_id || "—"}
Time
{formatDateTime(selected.created_at)}
{JSON.stringify(selected.details, null, 2)}
} - - - ); -} diff --git a/webui/src/features/admin/AdminOverviewPanel.tsx b/webui/src/features/admin/AdminOverviewPanel.tsx deleted file mode 100644 index b735b84..0000000 --- a/webui/src/features/admin/AdminOverviewPanel.tsx +++ /dev/null @@ -1,93 +0,0 @@ -import { useEffect, useState } from "react"; -import type { ApiSettings } from "../../types"; -import { fetchAdminOverview, type AdminOverview } from "../../api/admin"; -import Card from "../../components/Card"; -import Button from "../../components/Button"; -import AdminPageLayout from "./components/AdminPageLayout"; -import { adminErrorMessage } from "./adminUtils"; - -export default function AdminOverviewPanel({ settings, onSelect, availableSections }: { settings: ApiSettings; onSelect: (section: string) => void; availableSections: ReadonlySet }) { - const [overview, setOverview] = useState(null); - const [error, setError] = useState(""); - const [loading, setLoading] = useState(true); - - async function load() { - setLoading(true); - setError(""); - try { setOverview(await fetchAdminOverview(settings)); } - catch (err) { setError(adminErrorMessage(err)); } - finally { setLoading(false); } - } - - useEffect(() => { void load(); }, [settings.accessToken, settings.apiBaseUrl]); - - const hasSystemMetrics = Boolean(overview && [ - overview.tenant_count, - overview.system_account_count, - overview.system_group_template_count, - overview.system_role_template_count - ].some((value) => value !== null && value !== undefined)); - - return ( - void load()} disabled={loading}>Reload}> - {overview && <> - {hasSystemMetrics && <> -
System
-
- - - - -
- } - - {hasSystemArea(availableSections) && -
- {availableSections.has("system-settings") && onSelect("system-settings")} />} - {availableSections.has("system-tenants") && onSelect("system-tenants")} />} - {availableSections.has("system-roles") && onSelect("system-roles")} />} - {availableSections.has("system-role-templates") && onSelect("system-role-templates")} />} - {availableSections.has("system-groups") && onSelect("system-groups")} />} - {availableSections.has("system-users") && onSelect("system-users")} />} - {availableSections.has("system-mail-servers") && onSelect("system-mail-servers")} />} - {availableSections.has("system-retention") && onSelect("system-retention")} />} - {availableSections.has("system-audit") && onSelect("system-audit")} />} -
- } - -
Active tenant: {overview.active_tenant_name}
-
- - - - -
- - -
- {availableSections.has("tenant-settings") && onSelect("tenant-settings")} />} - {availableSections.has("tenant-roles") && onSelect("tenant-roles")} />} - {availableSections.has("tenant-groups") && onSelect("tenant-groups")} />} - {availableSections.has("tenant-users") && onSelect("tenant-users")} />} - {availableSections.has("tenant-mail-servers") && onSelect("tenant-mail-servers")} />} - {availableSections.has("tenant-retention") && onSelect("tenant-retention")} />} - {availableSections.has("tenant-api-keys") && onSelect("tenant-api-keys")} />} - {availableSections.has("tenant-audit") && onSelect("tenant-audit")} />} -
- - } - - ); -} - -function hasSystemArea(sections: ReadonlySet): boolean { - return ["system-settings", "system-tenants", "system-roles", "system-role-templates", "system-groups", "system-users", "system-mail-servers", "system-retention", "system-audit"].some((section) => sections.has(section)); -} - -function Metric({ title, value, text }: { title: string; value: string | number; text: string }) { - return {value}

{text}

; -} - -function AreaLink({ title, text, onClick }: { title: string; text: string; onClick: () => void }) { - return ; -} diff --git a/webui/src/features/admin/AdminPage.tsx b/webui/src/features/admin/AdminPage.tsx deleted file mode 100644 index 9267b84..0000000 --- a/webui/src/features/admin/AdminPage.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { useEffect, useMemo } from "react"; -import { useSearchParams } from "react-router-dom"; -import type { ApiSettings, AuthInfo, MailProfilesUiCapability } from "../../types"; -import { fetchMe } from "../../api/auth"; -import Card from "../../components/Card"; -import ModuleSubnav, { type ModuleSubnavGroup } from "../../layout/ModuleSubnav"; -import { adminReadScopes, hasAnyScope, hasScope } from "../../utils/permissions"; -import AdminOverviewPanel from "./AdminOverviewPanel"; -import SystemUsersPanel from "./SystemUsersPanel"; -import SystemSettingsPanel from "./SystemSettingsPanel"; -import TenantSettingsPanel from "./TenantSettingsPanel"; -import GovernanceTemplatesPanel from "./GovernanceTemplatesPanel"; -import SystemRolesPanel from "./SystemRolesPanel"; -import TenantsPanel from "./TenantsPanel"; -import UsersPanel from "./UsersPanel"; -import GroupsPanel from "./GroupsPanel"; -import RolesPanel from "./RolesPanel"; -import ApiKeysPanel from "./ApiKeysPanel"; -import AdminAuditPanel from "./AdminAuditPanel"; -import MailProfilesPanel from "./MailProfilesPanel"; -import RetentionPoliciesPanel from "./RetentionPoliciesPanel"; -import { usePlatformUiCapability } from "../../platform/ModuleContext"; - -type AdminSection = - | "overview" - | "system-settings" - | "system-retention" - | "system-mail-servers" - | "system-tenants" - | "system-users" - | "system-groups" - | "system-roles" - | "system-role-templates" - | "system-audit" - | "tenant-users" - | "tenant-groups" - | "tenant-roles" - | "tenant-api-keys" - | "tenant-mail-servers" - | "tenant-user-mail-servers" - | "tenant-group-mail-servers" - | "tenant-retention" - | "tenant-user-retention" - | "tenant-group-retention" - | "tenant-settings" - | "tenant-audit"; - -export default function AdminPage({ - settings, - auth, - onAuthChange -}: { - settings: ApiSettings; - auth: AuthInfo; - onAuthChange: (auth: AuthInfo | null, accessToken?: string) => void; -}) { - const mailProfilesUi = usePlatformUiCapability("mail.profiles"); - const mailProfilesAvailable = Boolean(mailProfilesUi); - - const available = useMemo(() => { - const sections = new Set(["overview"]); - if (hasScope(auth, "system:settings:read")) { - sections.add("system-settings"); - sections.add("system-retention"); - if (mailProfilesAvailable) sections.add("system-mail-servers"); - } - if (hasScope(auth, "system:tenants:read")) sections.add("system-tenants"); - if (hasAnyScope(auth, ["system:accounts:read", "system:access:read"])) sections.add("system-users"); - if (hasScope(auth, "system:governance:read")) sections.add("system-groups"); - if (hasAnyScope(auth, ["system:roles:read", "system:access:read"])) sections.add("system-roles"); - if (hasScope(auth, "system:governance:read")) sections.add("system-role-templates"); - if (hasScope(auth, "system:audit:read")) sections.add("system-audit"); - if (hasScope(auth, "admin:users:read")) sections.add("tenant-users"); - if (hasScope(auth, "admin:groups:read")) sections.add("tenant-groups"); - if (hasScope(auth, "admin:roles:read")) sections.add("tenant-roles"); - if (hasScope(auth, "admin:api_keys:read")) sections.add("tenant-api-keys"); - if (mailProfilesAvailable && hasAnyScope(auth, ["mail_servers:read", "admin:policies:read"])) { - sections.add("tenant-mail-servers"); - if (hasScope(auth, "admin:users:read")) sections.add("tenant-user-mail-servers"); - if (hasScope(auth, "admin:groups:read")) sections.add("tenant-group-mail-servers"); - } - if (hasScope(auth, "admin:policies:read")) { - sections.add("tenant-retention"); - if (hasScope(auth, "admin:users:read")) sections.add("tenant-user-retention"); - if (hasScope(auth, "admin:groups:read")) sections.add("tenant-group-retention"); - } - if (hasScope(auth, "admin:settings:read")) sections.add("tenant-settings"); - if (hasScope(auth, "audit:read")) sections.add("tenant-audit"); - return sections; - }, [auth, mailProfilesAvailable]); - const [searchParams, setSearchParams] = useSearchParams(); - const requestedSection = searchParams.get("section") as AdminSection | null; - const active: AdminSection = requestedSection && available.has(requestedSection) ? requestedSection : "overview"; - - function selectSection(section: AdminSection) { - const next = new URLSearchParams(searchParams); - if (section === "overview") next.delete("section"); - else next.set("section", section); - setSearchParams(next, { replace: true }); - } - - useEffect(() => { - if (requestedSection && !available.has(requestedSection)) selectSection("overview"); - }, [requestedSection, available]); - - async function refreshAuth() { - onAuthChange(await fetchMe(settings)); - } - - useEffect(() => { - void refreshAuth().catch(() => undefined); - }, [settings.accessToken, settings.apiBaseUrl]); - - if (!hasAnyScope(auth, adminReadScopes)) { - return

Your current roles do not grant administrative access.

; - } - - const adminSubnav: ModuleSubnavGroup[] = [ - { items: [{ id: "overview" as const, label: "Overview" }] }, - { - title: "SYSTEM", - items: [ - ...(available.has("system-settings") ? [{ id: "system-settings" as const, label: "General" }] : []), - ...(available.has("system-tenants") ? [{ id: "system-tenants" as const, label: "Tenants" }] : []), - ...(available.has("system-roles") ? [{ id: "system-roles" as const, label: "System roles" }] : []), - ...(available.has("system-role-templates") ? [{ id: "system-role-templates" as const, label: "Tenant roles" }] : []), - ...(available.has("system-groups") ? [{ id: "system-groups" as const, label: "Groups" }] : []), - ...(available.has("system-users") ? [{ id: "system-users" as const, label: "Users" }] : []), - ...(available.has("system-mail-servers") ? [{ id: "system-mail-servers" as const, label: "Mail servers" }] : []), - ...(available.has("system-retention") ? [{ id: "system-retention" as const, label: "Retention" }] : []), - ...(available.has("system-audit") ? [{ id: "system-audit" as const, label: "Audit" }] : []) - ] - }, - { - title: "TENANT", - items: [ - ...(available.has("tenant-settings") ? [{ id: "tenant-settings" as const, label: "General" }] : []), - ...(available.has("tenant-roles") ? [{ id: "tenant-roles" as const, label: "Roles" }] : []), - ...(available.has("tenant-groups") ? [{ id: "tenant-groups" as const, label: "Groups" }] : []), - ...(available.has("tenant-users") ? [{ id: "tenant-users" as const, label: "Users" }] : []), - ...(available.has("tenant-mail-servers") ? [{ id: "tenant-mail-servers" as const, label: "Mail servers" }] : []), - ...(available.has("tenant-retention") ? [{ id: "tenant-retention" as const, label: "Retention" }] : []), - ...(available.has("tenant-api-keys") ? [{ id: "tenant-api-keys" as const, label: "API keys" }] : []), - ...(available.has("tenant-audit") ? [{ id: "tenant-audit" as const, label: "Audit" }] : []) - ] - }, - { - title: "GROUP", - items: [ - ...(available.has("tenant-group-mail-servers") ? [{ id: "tenant-group-mail-servers" as const, label: "Mail servers" }] : []), - ...(available.has("tenant-group-retention") ? [{ id: "tenant-group-retention" as const, label: "Retention" }] : []), - ] - }, - { - title: "USER", - items: [ - ...(available.has("tenant-user-mail-servers") ? [{ id: "tenant-user-mail-servers" as const, label: "Mail servers" }] : []), - ...(available.has("tenant-user-retention") ? [{ id: "tenant-user-retention" as const, label: "Retention" }] : []), - ] - } - ].filter((group) => group.items.length > 0); - - return ( -
- -
-
- {active === "overview" && available.has(section as AdminSection) && selectSection(section as AdminSection)} />} - {active === "system-settings" && } - {active === "system-retention" && } - {active === "system-mail-servers" && } - {active === "system-tenants" && } - {active === "system-users" && } - {active === "system-groups" && } - {active === "system-roles" && } - {active === "system-role-templates" && } - {active === "system-audit" && } - {active === "tenant-users" && } - {active === "tenant-groups" && } - {active === "tenant-roles" && } - {active === "tenant-api-keys" && } - {active === "tenant-mail-servers" && } - {active === "tenant-user-mail-servers" && } - {active === "tenant-group-mail-servers" && } - {active === "tenant-retention" && } - {active === "tenant-user-retention" && } - {active === "tenant-group-retention" && } - {active === "tenant-settings" && } - {active === "tenant-audit" && } -
-
-
- ); -} diff --git a/webui/src/features/admin/ApiKeysPanel.tsx b/webui/src/features/admin/ApiKeysPanel.tsx deleted file mode 100644 index ac2d530..0000000 --- a/webui/src/features/admin/ApiKeysPanel.tsx +++ /dev/null @@ -1,127 +0,0 @@ -import { useEffect, useMemo, useState } from "react"; -import { Pencil, Plus, Search, Trash2 } from "lucide-react"; -import type { ApiSettings, AuthInfo } from "../../types"; -import { createApiKey, fetchApiKeys, fetchPermissionCatalog, fetchUsers, revokeApiKey, type ApiKeyAdminItem, type PermissionItem, type UserAdminItem } from "../../api/admin"; -import Button from "../../components/Button"; -import DataGrid, { type DataGridColumn } from "../../components/table/DataGrid"; -import Dialog from "../../components/Dialog"; -import FormField from "../../components/FormField"; -import StatusBadge from "../../components/StatusBadge"; -import ConfirmDialog from "../../components/ConfirmDialog"; -import AdminSelectionList from "./components/AdminSelectionList"; -import AdminIconButton from "./components/AdminIconButton"; -import AdminPageLayout from "./components/AdminPageLayout"; -import { adminErrorMessage, formatDateTime } from "./adminUtils"; -import { scopeGrants } from "../../utils/permissions"; - -export default function ApiKeysPanel({ settings, auth, canCreate, canRevoke }: { settings: ApiSettings; auth: AuthInfo; canCreate: boolean; canRevoke: boolean }) { - const [keys, setKeys] = useState([]); - const [users, setUsers] = useState([]); - const [permissions, setPermissions] = useState([]); - const [showRevoked, setShowRevoked] = useState(false); - const [creating, setCreating] = useState(false); - const [viewing, setViewing] = useState(null); - const [draft, setDraft] = useState({ name: "", userId: auth.user.id, scopes: ["campaign:read"], expiresAt: "" }); - const [secret, setSecret] = useState<{ name: string; value: string } | null>(null); - const [revoking, setRevoking] = useState(null); - const [loading, setLoading] = useState(true); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(""); - const [success, setSuccess] = useState(""); - - async function load() { - setLoading(true); - setError(""); - try { - const [nextKeys, nextUsers, nextPermissions] = await Promise.all([fetchApiKeys(settings, showRevoked), fetchUsers(settings), fetchPermissionCatalog(settings)]); - setKeys(nextKeys); - setUsers(nextUsers.filter((user) => user.is_active && user.account_is_active)); - setPermissions(nextPermissions.filter((permission) => permission.level === "tenant")); - } catch (err) { setError(adminErrorMessage(err)); } - finally { setLoading(false); } - } - - useEffect(() => { void load(); }, [settings.accessToken, settings.apiBaseUrl, (auth.active_tenant ?? auth.tenant).id, showRevoked]); - - const selectedUser = users.find((user) => user.id === draft.userId); - const allowedPermissions = permissions.filter((permission) => selectedUser?.effective_scopes.some((scope) => scopeGrants(scope, permission.scope))); - - const columns = useMemo[]>(() => [ - { id: "name", header: "Name", width: "minmax(190px, 1fr)", minWidth: 170, resizable: true, sticky: "start", sortable: true, filterable: true, value: (row) => row.name, render: (row) =>
{row.name}
{row.prefix}…
}, - { id: "owner", header: "Owner", width: 250, minWidth: 180, maxWidth: 480, resizable: true, fill: true, sortable: true, filterable: true, value: (row) => row.user_email }, - { id: "scopes", header: "Scopes", width: 120, resizable: false, sortable: true, filterable: true, filterType: "integer", value: (row) => row.scopes.length, render: (row) => String(row.scopes.length) }, - { id: "status", header: "Status", width: 120, resizable: false, sortable: true, filterable: true, value: (row) => row.revoked_at ? "revoked" : "active", render: (row) => }, - { id: "last_used", header: "Last used", width: 180, minWidth: 150, resizable: true, sortable: true, value: (row) => row.last_used_at || "", render: (row) => formatDateTime(row.last_used_at) }, - { id: "expires", header: "Expires", width: 180, minWidth: 150, resizable: true, sortable: true, value: (row) => row.expires_at || "", render: (row) => row.expires_at ? formatDateTime(row.expires_at) : "No expiry" }, - { id: "actions", header: "Actions", width: 150, sticky: "end", resizable: false, align: "right", render: (row) =>
- } onClick={() => setViewing(row)} /> - } disabled /> - } variant="danger" onClick={() => setRevoking(row)} disabled={!canRevoke || Boolean(row.revoked_at)} /> -
} - ], [canRevoke]); - - function openCreate() { - const defaultUser = users.find((user) => user.id === auth.user.id) ?? users[0]; - setDraft({ name: "", userId: defaultUser?.id || "", scopes: ["campaign:read"], expiresAt: "" }); - setCreating(true); - setError(""); - } - - async function save() { - setBusy(true); - setError(""); - try { - const created = await createApiKey(settings, { name: draft.name, user_id: draft.userId, scopes: draft.scopes, expires_at: draft.expiresAt ? new Date(draft.expiresAt).toISOString() : null }); - setSecret({ name: created.name, value: created.secret }); - setCreating(false); - setSuccess(`API key ${created.name} created.`); - await load(); - } catch (err) { setError(adminErrorMessage(err)); } - finally { setBusy(false); } - } - - async function revoke() { - if (!revoking) return; - setBusy(true); - setError(""); - try { - await revokeApiKey(settings, revoking.id); - setSuccess(`API key ${revoking.name} revoked.`); - setRevoking(null); - await load(); - } catch (err) { setError(adminErrorMessage(err)); } - finally { setBusy(false); } - } - - return ( - <> - } variant="primary" onClick={openCreate} disabled={!canCreate || !users.length} />}> -
row.id} emptyText="No API keys found." />
- - - !busy && setCreating(false)} className="admin-dialog admin-dialog-wide" footer={<>}> -
- setDraft({ ...draft, name: event.target.value })} /> - - setDraft({ ...draft, expiresAt: event.target.value })} /> -
-
Allowed scopes ({ id: permission.scope, label: permission.label, description: `${permission.scope} — ${permission.description}` }))} selected={draft.scopes} onChange={(scopes) => setDraft({ ...draft, scopes })} emptyText="The selected user has no tenant permissions available to an API key." />
- - - setViewing(null)} className="admin-dialog admin-dialog-wide" footer={}> - {viewing && <>
-
Name
{viewing.name}
Prefix
{viewing.prefix}…
-
Owner
{viewing.user_email}
Status
{viewing.revoked_at ? "Revoked" : "Active"}
-
Created
{formatDateTime(viewing.created_at)}
Last used
{formatDateTime(viewing.last_used_at)}
-
Expires
{viewing.expires_at ? formatDateTime(viewing.expires_at) : "No expiry"}
Revoked
{formatDateTime(viewing.revoked_at)}
-

Scopes

{viewing.scopes.map((scope) => {scope})}
} - - - setSecret(null)} className="admin-dialog" footer={}> - {secret && <>

The secret for {secret.name} is shown once.

{secret.value}

Store it in a secret manager. Only its prefix and hash remain in Multi Seal Mail.

} - - - setRevoking(null)} onConfirm={() => void revoke()} /> - - ); -} diff --git a/webui/src/features/admin/GovernanceTemplatesPanel.tsx b/webui/src/features/admin/GovernanceTemplatesPanel.tsx deleted file mode 100644 index 37ee38d..0000000 --- a/webui/src/features/admin/GovernanceTemplatesPanel.tsx +++ /dev/null @@ -1,232 +0,0 @@ -import { useEffect, useMemo, useState } from "react"; -import { Search, Pencil, Plus, Trash2 } from "lucide-react"; -import type { ApiSettings } from "../../types"; -import Button from "../../components/Button"; -import ConfirmDialog from "../../components/ConfirmDialog"; -import DataGrid, { type DataGridColumn } from "../../components/table/DataGrid"; -import Dialog from "../../components/Dialog"; -import FormField from "../../components/FormField"; -import StatusBadge from "../../components/StatusBadge"; -import { - createGovernanceTemplate, - deleteGovernanceTemplate, - fetchGovernanceTemplates, - fetchPermissionCatalog, - fetchTenants, - updateGovernanceTemplate, - type GovernanceAssignment, - type GovernanceTemplateItem, - type PermissionItem, - type TenantAdminItem -} from "../../api/admin"; -import AdminIconButton from "./components/AdminIconButton"; -import AdminPageLayout from "./components/AdminPageLayout"; -import AdminSelectionList from "./components/AdminSelectionList"; -import { adminErrorMessage, joinLabels } from "./adminUtils"; - -const emptyDraft = { - slug: "", - name: "", - description: "", - isActive: true, - permissions: [] as string[], - assignments: [] as GovernanceAssignment[] -}; - -export default function GovernanceTemplatesPanel({ - settings, - kind, - canWrite, - onAuthRefresh -}: { - settings: ApiSettings; - kind: "group" | "role"; - canWrite: boolean; - onAuthRefresh: () => Promise; -}) { - const [items, setItems] = useState([]); - const [tenants, setTenants] = useState([]); - const [permissions, setPermissions] = useState([]); - const [editing, setEditing] = useState(null); - const [viewing, setViewing] = useState(null); - const [deleting, setDeleting] = useState(null); - const [draft, setDraft] = useState(emptyDraft); - const [loading, setLoading] = useState(true); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(""); - const [success, setSuccess] = useState(""); - - async function load() { - setLoading(true); - setError(""); - try { - const [nextItems, nextTenants, nextPermissions] = await Promise.all([ - fetchGovernanceTemplates(settings, kind), - fetchTenants(settings), - kind === "role" ? fetchPermissionCatalog(settings) : Promise.resolve([]) - ]); - setItems(nextItems); - setTenants(nextTenants); - setPermissions(nextPermissions.filter((item) => item.level === "tenant")); - } catch (err) { setError(adminErrorMessage(err)); } - finally { setLoading(false); } - } - - useEffect(() => { void load(); }, [settings.accessToken, settings.apiBaseUrl, kind]); - - function openCreate() { - setDraft(emptyDraft); - setEditing("new"); - } - - function openEdit(item: GovernanceTemplateItem) { - setDraft({ - slug: item.slug, - name: item.name, - description: item.description || "", - isActive: item.is_active, - permissions: item.permissions, - assignments: item.assignments - }); - setEditing(item); - } - - function assignmentMode(tenantId: string): "none" | "available" | "required" { - return draft.assignments.find((item) => item.tenant_id === tenantId)?.mode ?? "none"; - } - - function setAssignment(tenantId: string, mode: "none" | "available" | "required") { - setDraft((current) => ({ - ...current, - assignments: mode === "none" - ? current.assignments.filter((item) => item.tenant_id !== tenantId) - : [...current.assignments.filter((item) => item.tenant_id !== tenantId), { tenant_id: tenantId, mode }] - })); - } - - async function save() { - setBusy(true); - setError(""); - try { - if (editing === "new") { - await createGovernanceTemplate(settings, { - kind, - slug: draft.slug, - name: draft.name, - description: draft.description || null, - permissions: kind === "role" ? draft.permissions : [], - is_active: draft.isActive, - assignments: draft.assignments - }); - setSuccess(`${kind === "group" ? "Group" : "Role"} template created.`); - } else if (editing) { - await updateGovernanceTemplate(settings, editing.id, { - name: draft.name, - description: draft.description || null, - permissions: kind === "role" ? draft.permissions : [], - is_active: draft.isActive, - assignments: draft.assignments - }); - setSuccess(`${draft.name} updated and synchronized to assigned tenants.`); - } - setEditing(null); - await load(); - await onAuthRefresh(); - } catch (err) { setError(adminErrorMessage(err)); } - finally { setBusy(false); } - } - - async function remove() { - if (!deleting) return; - setBusy(true); - setError(""); - try { - await deleteGovernanceTemplate(settings, deleting.id); - setSuccess(`${deleting.name} deleted.`); - setDeleting(null); - await load(); - await onAuthRefresh(); - } catch (err) { setError(adminErrorMessage(err)); } - finally { setBusy(false); } - } - - const columns = useMemo[]>(() => [ - { - id: "template", header: kind === "group" ? "Group template" : "Tenant role", width: "minmax(240px, 1.2fr)", minWidth: 210, resizable: true, sticky: "start", sortable: true, filterable: true, - value: (row) => `${row.name} ${row.slug}`, - render: (row) =>
{row.name}
{row.slug}
- }, - { - id: "tenants", header: "Tenant availability", width: 320, minWidth: 210, maxWidth: 640, resizable: true, fill: true, sortable: true, filterable: true, - value: (row) => row.assignments.map((assignment) => tenants.find((tenant) => tenant.id === assignment.tenant_id)?.name || assignment.tenant_id).join(", ") || "—", - render: (row) => row.assignments.length ? row.assignments.map((assignment) => { - const tenant = tenants.find((item) => item.id === assignment.tenant_id); - return `${tenant?.name || assignment.tenant_id} (${assignment.mode})`; - }).join(", ") : "—" - }, - ...(kind === "role" ? [{ - id: "permissions", header: "Permissions", width: 120, resizable: false, sortable: true, filterable: true, filterType: "integer" as const, - value: (row: GovernanceTemplateItem) => row.effective_permission_count - }] : []), - { id: "status", header: "Status", width: 120, resizable: false, sortable: true, filterable: true, value: (row) => row.is_active ? "active" : "inactive", render: (row) => }, - { - id: "actions", header: "Actions", width: 150, sticky: "end", resizable: false, align: "right", - render: (row) =>
- } onClick={() => setViewing(row)} /> - } onClick={() => openEdit(row)} disabled={!canWrite} /> - } variant="danger" onClick={() => setDeleting(row)} disabled={!canWrite} /> -
- } - ], [canWrite, kind, tenants]); - - const title = kind === "group" ? "Central groups" : "Tenant roles"; - const description = kind === "group" - ? "Centrally defined group identities that are provisioned into selected tenants as available or required definitions. Membership remains tenant-local." - : "Centrally governed tenant roles that are provisioned into selected tenants as available or required definitions."; - - return ( - <> - } variant="primary" onClick={openCreate} disabled={!canWrite} />} - > -
- row.id} emptyText={`No central ${kind} templates found.`} /> -
- - - !busy && setEditing(null)} - className="admin-dialog admin-dialog-wide" - footer={<>} - > -
- setDraft({ ...draft, name: event.target.value })} /> - setDraft({ ...draft, slug: event.target.value })} /> - -