From 7192d32e65c6205d1084f15088c3033ed0b725a3 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 1 Aug 2026 17:46:54 +0200 Subject: [PATCH] feat: add institutional governance and recovery contracts --- AGENTS.md | 2 + ...b8c2d6f90_runtime_coordination_recovery.py | 24 + ...b8c2d6f90_runtime_coordination_recovery.py | 232 ++ docs/ACTION_EFFECT_AUTOMATION_LAYER.md | 16 + docs/CODEX_WORKFLOW.md | 10 + docs/CONFIGURATION_PACKAGES.md | 32 + docs/DOCUMENTATION_MAP.md | 4 + ...NAL_REFERENCES_AND_INTEGRATION_MATURITY.md | 20 + docs/GOVOPLAN_MASTER_ROADMAP.md | 129 +- docs/INSTITUTIONAL_CONTEXT_CONTRACT.md | 159 + docs/MODULE_ARCHITECTURE.md | 150 +- docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md | 6 +- docs/STATE_AND_RECOVERY_CONTRACT.md | 143 + pyproject.toml | 1 + src/govoplan_core/audit/logging.py | 27 +- src/govoplan_core/celery_app.py | 205 +- src/govoplan_core/commands/fenced_run.py | 199 + src/govoplan_core/commands/init_db.py | 51 +- .../commands/wait_for_database.py | 61 + src/govoplan_core/core/access.py | 2 + src/govoplan_core/core/automation.py | 2 + .../core/configuration_packages.py | 560 ++- src/govoplan_core/core/datasources.py | 179 + src/govoplan_core/core/events.py | 9 + src/govoplan_core/core/external_references.py | 38 + src/govoplan_core/core/install_config.py | 479 ++- src/govoplan_core/core/institutional.py | 3310 +++++++++++++++++ src/govoplan_core/core/module_installer.py | 12 + .../core/module_package_catalog.py | 72 + src/govoplan_core/core/modules.py | 11 + src/govoplan_core/core/object_storage.py | 597 +++ src/govoplan_core/core/provider_governance.py | 1376 +++++++ src/govoplan_core/core/recovery.py | 669 ++++ src/govoplan_core/core/registry.py | 184 + .../core/runtime_coordination.py | 605 +++ src/govoplan_core/db/bootstrap.py | 2 + src/govoplan_core/db/migration_lock.py | 77 + src/govoplan_core/db/migrations.py | 47 + src/govoplan_core/server/default_config.py | 63 +- src/govoplan_core/server/platform.py | 9 + src/govoplan_core/server/runtime_agent.py | 129 + src/govoplan_core/settings.py | 38 + ...test_configuration_package_architecture.py | 242 ++ tests/test_external_reference_contract.py | 15 + tests/test_fenced_run.py | 58 + tests/test_install_config.py | 43 + tests/test_institutional_contract.py | 816 ++++ tests/test_migration_lock.py | 55 + tests/test_module_system.py | 35 + tests/test_object_storage.py | 169 + tests/test_provider_governance_contract.py | 536 +++ tests/test_recovery_guarantees.py | 288 ++ tests/test_runtime_agents.py | 125 + tests/test_runtime_coordination.py | 130 + tests/test_wait_for_database.py | 43 + webui/package-lock.json | 146 + webui/package.json | 7 + webui/scripts/test-module-permutations.mjs | 16 +- webui/src/platform/modules.ts | 6 +- webui/src/types.ts | 25 + webui/vite.config.ts | 11 + 61 files changed, 12539 insertions(+), 168 deletions(-) create mode 100644 alembic/dev_versions/e14b8c2d6f90_runtime_coordination_recovery.py create mode 100644 alembic/versions/e14b8c2d6f90_runtime_coordination_recovery.py create mode 100644 docs/INSTITUTIONAL_CONTEXT_CONTRACT.md create mode 100644 docs/STATE_AND_RECOVERY_CONTRACT.md create mode 100644 src/govoplan_core/commands/fenced_run.py create mode 100644 src/govoplan_core/commands/wait_for_database.py create mode 100644 src/govoplan_core/core/institutional.py create mode 100644 src/govoplan_core/core/object_storage.py create mode 100644 src/govoplan_core/core/provider_governance.py create mode 100644 src/govoplan_core/core/recovery.py create mode 100644 src/govoplan_core/core/runtime_coordination.py create mode 100644 src/govoplan_core/db/migration_lock.py create mode 100644 src/govoplan_core/server/runtime_agent.py create mode 100644 tests/test_configuration_package_architecture.py create mode 100644 tests/test_fenced_run.py create mode 100644 tests/test_institutional_contract.py create mode 100644 tests/test_migration_lock.py create mode 100644 tests/test_object_storage.py create mode 100644 tests/test_provider_governance_contract.py create mode 100644 tests/test_recovery_guarantees.py create mode 100644 tests/test_runtime_agents.py create mode 100644 tests/test_runtime_coordination.py create mode 100644 tests/test_wait_for_database.py diff --git a/AGENTS.md b/AGENTS.md index 663349a..aad4d93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,8 @@ tools/checks/check-focused.sh - Prefer `rg`, `sed`, and targeted tests over broad recursive scans or full builds. - Avoid DataGrid changes unless explicitly requested; it is intentionally brittle and has known deferred work. - Do not add module-to-module imports for optional integrations. Use core registry/capability/module metadata paths. +- Treat documentation as part of every behavior change. Update the owning module's manifest-driven `DocumentationTopic` contributions for each affected user and administrator workflow, setting, permission, limitation, and operational consequence. Feature modules own this content; `govoplan-docs` projects it and must not import feature internals. +- Keep a static user and administrator documentation baseline in every module manifest, even when richer configured-state topics come from `documentation_providers`. Run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after changing a manifest or module behavior. - Treat Gitea issues as the canonical backlog and state log. Treat Gitea wiki pages as durable project context mirrored from repository and product docs. Use `docs/GITEA_ISSUES.md` for labels, templates, TODO import, wiki sync, and Codex issue updates. - Do not keep generated WebUI test folders in git status; they should be ignored and removable. - Do not start persistent dev servers unless the user asks. diff --git a/alembic/dev_versions/e14b8c2d6f90_runtime_coordination_recovery.py b/alembic/dev_versions/e14b8c2d6f90_runtime_coordination_recovery.py new file mode 100644 index 0000000..4448312 --- /dev/null +++ b/alembic/dev_versions/e14b8c2d6f90_runtime_coordination_recovery.py @@ -0,0 +1,24 @@ +"""development-track wrapper for runtime coordination and recovery.""" +from __future__ import annotations + +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + + +_path = ( + Path(__file__).resolve().parents[1] + / "versions" + / "e14b8c2d6f90_runtime_coordination_recovery.py" +) +_spec = spec_from_file_location("govoplan_runtime_coordination_migration", _path) +if _spec is None or _spec.loader is None: + raise RuntimeError(f"Unable to load runtime coordination migration from {_path}") +_migration = module_from_spec(_spec) +_spec.loader.exec_module(_migration) + +revision = _migration.revision +down_revision = _migration.down_revision +branch_labels = _migration.branch_labels +depends_on = _migration.depends_on +upgrade = _migration.upgrade +downgrade = _migration.downgrade diff --git a/alembic/versions/e14b8c2d6f90_runtime_coordination_recovery.py b/alembic/versions/e14b8c2d6f90_runtime_coordination_recovery.py new file mode 100644 index 0000000..4063aa0 --- /dev/null +++ b/alembic/versions/e14b8c2d6f90_runtime_coordination_recovery.py @@ -0,0 +1,232 @@ +"""add runtime coordination and recovery evidence + +Revision ID: e14b8c2d6f90 +Revises: d03a7b9c1e5f +Create Date: 2026-08-01 00:00:00.000000 +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "e14b8c2d6f90" +down_revision = "d03a7b9c1e5f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + tables = set(inspector.get_table_names()) + if "core_runtime_nodes" not in tables: + op.create_table( + "core_runtime_nodes", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("installation_id", sa.String(length=100), nullable=False), + sa.Column("node_id", sa.String(length=200), nullable=False), + sa.Column("incarnation", sa.String(length=36), nullable=False), + sa.Column("role", sa.String(length=40), nullable=False), + sa.Column("software_version", sa.String(length=80), nullable=False), + sa.Column("composition_hash", sa.String(length=64), nullable=False), + sa.Column("queues", sa.JSON(), nullable=False), + sa.Column("state", sa.String(length=30), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("drain_requested_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("drain_reason", sa.String(length=500), nullable=True), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("metadata", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_core_runtime_nodes")), + sa.UniqueConstraint( + "installation_id", + "node_id", + name="uq_core_runtime_node_installation_node", + ), + ) + for column in ( + "installation_id", + "node_id", + "incarnation", + "role", + "composition_hash", + "state", + ): + op.create_index( + op.f(f"ix_core_runtime_nodes_{column}"), + "core_runtime_nodes", + [column], + unique=False, + ) + op.create_index( + "ix_core_runtime_nodes_installation_state_heartbeat", + "core_runtime_nodes", + ["installation_id", "state", "last_heartbeat_at"], + unique=False, + ) + + if "core_distributed_leases" not in tables: + op.create_table( + "core_distributed_leases", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("installation_id", sa.String(length=100), nullable=False), + sa.Column("resource_key", sa.String(length=255), nullable=False), + sa.Column("holder_node_id", sa.String(length=200), nullable=True), + sa.Column("holder_incarnation", sa.String(length=36), nullable=True), + sa.Column("fencing_token", sa.BigInteger(), nullable=False), + sa.Column("acquired_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("renewed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("metadata", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_core_distributed_leases")), + sa.UniqueConstraint( + "installation_id", + "resource_key", + name="uq_core_distributed_lease_resource", + ), + ) + for column in ( + "installation_id", + "resource_key", + "holder_node_id", + "holder_incarnation", + ): + op.create_index( + op.f(f"ix_core_distributed_leases_{column}"), + "core_distributed_leases", + [column], + unique=False, + ) + op.create_index( + "ix_core_distributed_leases_expiry", + "core_distributed_leases", + ["installation_id", "expires_at"], + unique=False, + ) + + if "core_recovery_operations" not in tables: + op.create_table( + "core_recovery_operations", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("installation_id", sa.String(length=100), nullable=False), + sa.Column("module_id", sa.String(length=100), nullable=False), + sa.Column("operation_type", sa.String(length=100), nullable=False), + sa.Column("resource_type", sa.String(length=100), nullable=True), + sa.Column("resource_id", sa.String(length=255), nullable=True), + sa.Column("mode", sa.String(length=40), nullable=False), + sa.Column("status", sa.String(length=40), nullable=False), + sa.Column("idempotency_key", sa.String(length=200), nullable=False), + sa.Column("request_sha256", sa.String(length=64), nullable=False), + sa.Column("plan", sa.JSON(), nullable=False), + sa.Column("backup_reference", sa.String(length=1000), nullable=True), + sa.Column("approval_reference", sa.String(length=1000), nullable=True), + sa.Column("lease_resource_key", sa.String(length=255), nullable=True), + sa.Column("holder_node_id", sa.String(length=200), nullable=True), + sa.Column("holder_incarnation", sa.String(length=36), nullable=True), + sa.Column("fencing_token", sa.BigInteger(), nullable=True), + sa.Column("checkpoint_count", sa.Integer(), nullable=False), + sa.Column("evidence_head_sha256", sa.String(length=64), nullable=True), + sa.Column("failure_summary", sa.Text(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("recovery_started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("recovered_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revision", sa.Integer(), nullable=False), + sa.Column("metadata", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_core_recovery_operations")), + sa.UniqueConstraint( + "installation_id", + "module_id", + "idempotency_key", + name="uq_core_recovery_operation_idempotency", + ), + ) + for column in ( + "installation_id", + "module_id", + "operation_type", + "resource_type", + "resource_id", + "mode", + "status", + ): + op.create_index( + op.f(f"ix_core_recovery_operations_{column}"), + "core_recovery_operations", + [column], + unique=False, + ) + op.create_index( + "ix_core_recovery_operations_status_updated", + "core_recovery_operations", + ["installation_id", "status", "updated_at"], + unique=False, + ) + op.create_index( + "ix_core_recovery_operations_resource", + "core_recovery_operations", + ["module_id", "resource_type", "resource_id"], + unique=False, + ) + + if "core_recovery_checkpoints" not in tables: + op.create_table( + "core_recovery_checkpoints", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("operation_id", sa.String(length=36), nullable=False), + sa.Column("sequence", sa.Integer(), nullable=False), + sa.Column("status", sa.String(length=40), nullable=False), + sa.Column("kind", sa.String(length=80), nullable=False), + sa.Column("summary", sa.Text(), nullable=False), + sa.Column("evidence", sa.JSON(), nullable=False), + sa.Column("previous_sha256", sa.String(length=64), nullable=True), + sa.Column("checkpoint_sha256", sa.String(length=64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["operation_id"], + ["core_recovery_operations.id"], + name=op.f( + "fk_core_recovery_checkpoints_operation_id_core_recovery_operations" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_core_recovery_checkpoints")), + sa.UniqueConstraint( + "operation_id", + "sequence", + name="uq_core_recovery_checkpoint_sequence", + ), + ) + for column in ("operation_id", "status", "checkpoint_sha256"): + op.create_index( + op.f(f"ix_core_recovery_checkpoints_{column}"), + "core_recovery_checkpoints", + [column], + unique=False, + ) + op.create_index( + "ix_core_recovery_checkpoints_operation_created", + "core_recovery_checkpoints", + ["operation_id", "created_at"], + unique=False, + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + tables = set(inspector.get_table_names()) + for table in ( + "core_recovery_checkpoints", + "core_recovery_operations", + "core_distributed_leases", + "core_runtime_nodes", + ): + if table in tables: + op.drop_table(table) diff --git a/docs/ACTION_EFFECT_AUTOMATION_LAYER.md b/docs/ACTION_EFFECT_AUTOMATION_LAYER.md index 7f6acf1..7d9f0ca 100644 --- a/docs/ACTION_EFFECT_AUTOMATION_LAYER.md +++ b/docs/ACTION_EFFECT_AUTOMATION_LAYER.md @@ -99,6 +99,22 @@ The runner must never advance workflow state past a required side effect unless the action definition explicitly allows asynchronous completion and the pending state is visible. +For external and asynchronous effects, providers must preserve the distinction +between: + +1. requested intent; +2. approved intent; +3. dispatched command; +4. possibly executed but unconfirmed outcome; +5. confirmed observed effect; +6. reconciled, corrected, or compensated outcome. + +An API timeout after dispatch is not a failed effect and must not be retried as +a fresh command. The actor context should retain the real identity/account, +represented function or party, delegation or power, and mandate/jurisdiction +references when applicable. Domain modules remain responsible for deciding +which of those references are required for their action. + ## Failure States Automation should use explicit failure states: diff --git a/docs/CODEX_WORKFLOW.md b/docs/CODEX_WORKFLOW.md index 61ee682..ae56d3b 100644 --- a/docs/CODEX_WORKFLOW.md +++ b/docs/CODEX_WORKFLOW.md @@ -49,6 +49,15 @@ The broad writable root reduces approval churn. The explicit project trust entri Each active repository has an `AGENTS.md` file. These files define ownership, module boundaries, and focused commands for Codex. Keep durable project conventions there instead of repeating them in every prompt. +Documentation is part of the completion criteria for every behavior change. The +owning module must update its manifest-driven `DocumentationTopic` contributions +for affected user and administrator workflows, settings, permissions, +limitations, and operational consequences. Feature documentation remains in the +feature module; the optional `govoplan-docs` module projects those contributions +without importing feature internals. Every module manifest must retain a static +user and administrator baseline even when runtime providers add configured-state +details. + Use `~/.codex/config.toml` for personal defaults, auth/runtime settings, writable roots, and trust decisions. Avoid checking in absolute-path writable roots or model preferences unless they are intentionally team-wide. Use Gitea issues as the canonical backlog and state log. See `docs/GITEA_ISSUES.md` for label setup, TODO import, and Codex issue update commands. Durable docs should describe stable behavior; changing work state belongs on the issue. @@ -81,5 +90,6 @@ PATH=/mnt/DATA/git/govoplan-core/webui/node_modules/.bin:/home/zemion/.nvm/versi - Avoid broad recursive scans and full builds unless the change warrants them. - Keep generated build/test folders ignored. - Keep optional module behavior behind core registry/capability/module metadata boundaries. +- Run `tools/checks/check-manifest-shapes.py` after module behavior or manifest changes so user/admin documentation coverage remains complete. - Create or update Gitea issues for TODOs, follow-ups, blockers, and feature requests instead of keeping local backlog files. - Do not start persistent dev servers unless the user asks. diff --git a/docs/CONFIGURATION_PACKAGES.md b/docs/CONFIGURATION_PACKAGES.md index 1cce041..e78eba2 100644 --- a/docs/CONFIGURATION_PACKAGES.md +++ b/docs/CONFIGURATION_PACKAGES.md @@ -48,6 +48,35 @@ interface = how configured parts connect data = what the operator must provide for this deployment ``` +## Package Classes + +The same signed package mechanism supports several explicitly named classes: + +| Class | Purpose | +| --- | --- | +| `reference` | Prove a bounded journey, degraded states, recovery, and target-environment acceptance. | +| `product` | Provide a reusable operating capability such as governed communication, service-to-decision, procurement/contracts, or governed BI. | +| `sector` | Specialize terminology, forms, rules, process baselines, controls, reports, and integrations for an institutional sector without forking modules. | +| `deployment` | Declare a supported infrastructure topology, operational assumptions, health gates, and recovery evidence. | +| `integration` | Declare external systems, provider bindings, source-authority modes, maturity, required health, and reconciliation behavior. | + +Package class is metadata and validation context, not additional authority. A +sector package does not become a module and cannot write another module's +tables. Packages may extend other packages only through versioned fragments and +must preserve provenance and parent constraints. + +The contract enforces class-specific evidence. Reference packages require +target, recovery, security, operations, accessibility, privacy, and +documentation evidence. Deployment and integration packages require their +corresponding target/recovery/operations evidence, while integration packages +also name provider authority and minimum-maturity expectations. Preflight +blocks a missing, incompatible, or unhealthy provider. A derived package may +tighten parent module, capability, and provider requirements but cannot remove +or loosen them. Every non-documentation claim made by reference, deployment, or +integration packages carries a `sha256:` binding. Repository checks +recompute those hashes, while signed package verification protects the declared +manifest during transport. + ## Package Model A configuration package should be a signed, portable manifest plus module-owned @@ -68,6 +97,9 @@ Required package metadata: - preflight checks and post-import health checks - migration or transformation rules for older package versions - provenance, export source metadata, and signature metadata +- package class and optional parent package/version constraints +- source-authority bindings and provider-operation expectations for every + external integration used by the package Portable configuration schemas follow `COMPATIBILITY_POLICY.md`: providers write only their current schema version and read that version plus the previous diff --git a/docs/DOCUMENTATION_MAP.md b/docs/DOCUMENTATION_MAP.md index fc83b5b..7fb8532 100644 --- a/docs/DOCUMENTATION_MAP.md +++ b/docs/DOCUMENTATION_MAP.md @@ -15,7 +15,10 @@ operator, and roadmap pages. | Policy decision DTOs and provenance | `POLICY_CONTRACTS.md` | Shared explain/provenance shape; module-specific policy docs should link here. | | Events and audit trace context | `EVENTS_AND_AUDIT.md` | Event dispatch semantics and audit payload conventions. | | Action/effect automation layer | `ACTION_EFFECT_AUTOMATION_LAYER.md` | Action/effect contracts, consequence preview, runner semantics, and module boundary for automation. | +| External references and integration maturity | `EXTERNAL_REFERENCES_AND_INTEGRATION_MATURITY.md` | Stable external identity and cumulative connector maturity; configured source authority is defined by the meta target architecture. | +| Institutional context and governed references | `INSTITUTIONAL_CONTEXT_CONTRACT.md` | Shared temporal, actor/representation, institution, mandate, service, party, decision, evidence, legal-basis, information-governance, presentation, and geo DTO/provider contracts. | | Postbox E2EE target architecture | `POSTBOX_E2EE_ARCHITECTURE.md` | Strategic encrypted postbox/mailbox model, key ownership, role mailbox semantics, and retraction limits. | +| Shared state, runtime coordination, and recovery | `STATE_AND_RECOVERY_CONTRACT.md` | State profiles, object storage, node registration/drain, fenced leases, migration ordering, and recovery evidence. | ## Release And Operations @@ -33,6 +36,7 @@ operator, and roadmap pages. | Topic | Canonical document | Notes | | --- | --- | --- | | Product roadmap and module routing | `GOVOPLAN_MASTER_ROADMAP.md` | Product-level sequencing, implementation gates, issue routing, and missing-module decisions. | +| Institutional governance target | `govoplan/docs/INSTITUTIONAL_GOVERNANCE_TARGET_ARCHITECTURE.md` | Cross-product semantic layers, source-authority modes, candidate Mandates/Services/Parties/Decisions boundaries, and migration sequence. | | UI/UX decisions | `UI_UX_DECISION_LEDGER.md` | Binding guided-UI decisions, open decisions, impact index, and review checklist. | | Interface ethics and design doctrine | `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md` | Product-level doctrine for context, decision, consequence, contestability, responsibility, and traceability. | | Public-sector integration posture | `PUBLIC_SECTOR_INTEGRATION_STRATEGY.md` | Strategy index; executable target inventory lives in `govoplan-connectors`. | diff --git a/docs/EXTERNAL_REFERENCES_AND_INTEGRATION_MATURITY.md b/docs/EXTERNAL_REFERENCES_AND_INTEGRATION_MATURITY.md index 5d3d749..4219d7a 100644 --- a/docs/EXTERNAL_REFERENCES_AND_INTEGRATION_MATURITY.md +++ b/docs/EXTERNAL_REFERENCES_AND_INTEGRATION_MATURITY.md @@ -35,6 +35,26 @@ Connectors must declare and document the maturity they actually implement. handling, deletion semantics, and observable failures. A link-only connector must not imply that GovOPlaN holds an authoritative copy. +## Source Authority Is A Separate Dimension + +Integration maturity states what an adapter is capable of doing. It does not +decide which system owns truth for a configured object or field group. A +binding separately selects one of the source-authority modes defined by the +[institutional governance target architecture](../../govoplan/docs/INSTITUTIONAL_GOVERNANCE_TARGET_ARCHITECTURE.md): + +- `native_authoritative` +- `external_authoritative` +- `external_mirror` +- `governed_sync` +- `governance_overlay` +- `linked_reference` + +A connector can therefore support `synchronize` while a tenant deliberately +uses it only as an external mirror. Conversely, a native GovOPlaN object may +retain link-only references to several external systems. Authority may be +narrowed by tenant, organization, service, object type, object, field group, or +process step and must be visible in provenance and configuration preflight. + ## Domain Ownership - Domain modules own native GovOPlaN objects and their authorization. diff --git a/docs/GOVOPLAN_MASTER_ROADMAP.md b/docs/GOVOPLAN_MASTER_ROADMAP.md index 1a440db..dae5584 100644 --- a/docs/GOVOPLAN_MASTER_ROADMAP.md +++ b/docs/GOVOPLAN_MASTER_ROADMAP.md @@ -16,6 +16,9 @@ service and operating configurations, connected outcome stories, and capability horizons. The selected five-stage delivery sequence and its gates are in the meta repository's [Reference Journey Program](https://git.add-ideas.de/GovOPlaN/govoplan/src/branch/main/docs/REFERENCE_JOURNEY_PROGRAM.md). +The semantic target, source-authority modes, and reconciliation with the +implemented platform are in the meta repository's +[Institutional Governance Target Architecture](https://git.add-ideas.de/GovOPlaN/govoplan/src/branch/main/docs/INSTITUTIONAL_GOVERNANCE_TARGET_ARCHITECTURE.md). Those product documents are canonical; this Core roadmap remains their technical sequencing and module-routing companion. @@ -64,8 +67,9 @@ verify or reverse those effects. `INTERFACE_ETHICS_AND_DESIGN_DOCTRINE.md`. - Automation must use governed action/effect contracts, not hidden side effects. The first automation layer is defined in - `ACTION_EFFECT_AUTOMATION_LAYER.md` and should start in `govoplan-workflow` - unless a separate automation module becomes justified. + `ACTION_EFFECT_AUTOMATION_LAYER.md`; its first runner lives in + `govoplan-workflow-engine`. Create a separate automation module only if the + scheduler/action runtime outgrows workflow coordination. - Encrypted postboxes are a strategic target. Early postbox, access, and identity-trust contracts should stay compatible with the E2EE architecture in `POSTBOX_E2EE_ARCHITECTURE.md`. @@ -144,8 +148,9 @@ pattern exists. | Structured forms and validation | `govoplan-forms` | | Uploaded files and managed storage | `govoplan-files` | | Case record and lifecycle | `govoplan-cases` | -| Workflow transitions and automation | `govoplan-workflow` | -| Action/effect catalogue and automation runner | first `govoplan-workflow`; possible future `govoplan-automation` if it outgrows workflow | +| Workflow runtime, transitions, and automation | `govoplan-workflow-engine` | +| Workflow definition editing | optional `govoplan-workflow` | +| Action/effect catalogue and automation runner | first `govoplan-workflow-engine`; possible future `govoplan-automation` only if it outgrows workflow | | Internal work queues and tasks | `govoplan-tasks` | | Appointment proposals and booking | `govoplan-appointments`, `govoplan-calendar` | | Postbox, email, and notifications | `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications` | @@ -153,13 +158,18 @@ pattern exists. | Organizational structures, units, and functions | `govoplan-organizations` | | Identity-to-function assignments and directory synchronization | `govoplan-idm` | | Identity trust, device keys, and encrypted postbox key contracts | `govoplan-identity-trust`, `govoplan-access`, `govoplan-postbox` | -| Service directory/catalog | `govoplan-portal` | +| Service directory presentation | `govoplan-portal` | +| Versioned institutional service definition | shared service contract first; candidate `govoplan-services` after reuse proof | +| Mandate, jurisdiction, and institutional authority | shared mandate contract first; candidate `govoplan-mandates` after reuse proof | +| Procedure-local parties and representation | shared party contract first; candidate `govoplan-parties` after reuse proof | +| Formal institutional decisions | shared decision contract first; candidate `govoplan-decisions` after reuse proof | | Permit/document generation | `govoplan-templates`, `govoplan-dms` | | Payment capture and accounting handoff | `govoplan-payments`, `govoplan-ledger` | | Authentication projection, roles, permissions, acting context | `govoplan-access` | | Tenants, policy, and audit | `govoplan-tenancy`, `govoplan-policy`, `govoplan-audit` | | External software integration | `govoplan-connectors` | -| Recurring extraction and transformation | possible future `govoplan-datasources`, possible future `govoplan-dataflow` | +| Governed data/register catalogue | `govoplan-datasources` | +| Recurring extraction and transformation | `govoplan-dataflow` over Datasources and Connectors contracts | | Reports, BI, and management visibility | `govoplan-reporting` | ## Configuration And Safety Target @@ -204,8 +214,9 @@ an editor applies a high-impact configuration change. ## Reference Journeys -The active sequence is selected. Workflow remains deliberately deferred and is -not a dependency of these journeys. +The active sequence is selected. Workflow Engine and its optional editor are +now available foundations, but a reference journey does not depend on Workflow +unless its package explicitly composes and proves it. ### Journey 1: Campaign Demonstration Composition @@ -256,9 +267,11 @@ The result must preserve official-key mappings, organizational and reporting date semantics, quality findings, quarantine/replay, transparent calculation, and reproducible promotion between development, test, and production. -Reporting consumes the product. Create `govoplan-datasources` or -`govoplan-dataflow` only after the concrete path proves repeated ownership that -does not belong to connectors, Reporting, or the producing domain module. +Reporting consumes the product. Datasources owns the governed source and +materialization lifecycle; Dataflow owns typed transformation/run lineage; +Connectors owns external transport. The concrete path must now prove those +implemented boundaries and expose any missing contracts instead of recreating +them inside Reporting or a producing domain module. ### Journey 5: Collaborative Document Lifecycle @@ -336,8 +349,9 @@ Create or refine in this order: access. 4. `govoplan-cases`: case record, status, assignments, deadlines, and case evidence. -5. `govoplan-workflow`: state machine, transitions, commands, and module - handoff. +5. `govoplan-workflow-engine`: state machine, transitions, commands, module + handoff, and resumable execution; optional `govoplan-workflow` supplies the + editor. 6. `govoplan-tasks`: work queues, assignments, due dates, and follow-ups. 7. `govoplan-templates`: permit/decision document generation. 8. `govoplan-postbox`, `govoplan-mail`, `govoplan-notifications`: applicant and @@ -526,31 +540,34 @@ Refine: dashboard data. - `govoplan-search`: permissioned cross-module discovery. -Create only when justified: +Refine the existing owners: -- `govoplan-datasources`: source catalog, connection profiles, schema discovery, - freshness, provenance. -- `govoplan-dataflow`: transformations, validation, lineage, scheduled runs, - publication outputs. -- `govoplan-projects`: only if OpenProject connectors and existing cases/tasks - cannot cover the required semantics. +- `govoplan-datasources`: governed data/register catalog, live/cached/static + sources, staging, immutable materializations, freshness, quality, legal and + organizational context, and provenance. Connector profiles and credentials + remain in Connectors. +- `govoplan-dataflow`: typed transformations, validation, lineage, manual, + scheduled and event-triggered runs, reusable definitions, and publication + outputs. +- `govoplan-projects`: native projects, portfolios, milestones, goals, + dependencies, capacity, outcomes, and external OpenProject references; + Connectors owns OpenProject transport and synchronization. Reference journey: monthly data extraction, transformation, validation, approval, publication, and reporting. -Recurring extraction/transformation should start as a configuration package -across connectors, files, workflow, reporting, and templates. The package should -register sources, declare schemas, define mapping/validation versions, schedule -runs, produce previewable diffs, write governed outputs, and preserve lineage, -hashes, operator actions, and audit evidence. Create `govoplan-datasources` or -`govoplan-dataflow` only after this work exposes repeated contracts that do not -belong to existing modules. +Recurring extraction/transformation should be delivered as a configuration +package across Connectors, Datasources, Dataflow, Workflow Engine, Reporting, +Files, and Templates. The package should register sources, declare schemas, +define mapping/validation versions, schedule runs, produce previewable diffs, +write governed outputs, and preserve lineage, hashes, operator actions, and +audit evidence. Exit criteria: - connector catalog exists before building many adapters -- dataflow is created only after recurring transformation becomes product - behavior +- datasource and dataflow ownership remains provider-neutral and is proved by + the recurring transformation package - reporting consumes governed sources with provenance ## Implementation Gates @@ -591,23 +608,25 @@ in the capability waves: 4. Implement one data-backed Templates/Reporting path and safe HIS-style deep launch. 5. Extend that concrete source into one governed university analytical data - product before generalizing data-source or dataflow ownership. + product and use it to harden the existing Datasources/Dataflow ownership, + quality, lineage, and promotion contracts. 6. Implement Files-backed DMS versions and one provider-neutral collaborative editing lifecycle, then connect Records handoff. 7. Maintain already integrated Calendar/Scheduling/Poll and other foundations; activate another capability cluster only when the current journey needs it or the product roadmap explicitly reprioritizes it. -8. Resume Workflow only by explicit product decision and constrain it with - stable actions from one demonstrated package. +8. Extend Workflow Engine and the optional editor only through stable actions + and one demonstrated package at a time. ## Deliberate Deferrals Defer these until a reference journey proves the need: - full ERP replacement -- native project management beyond connector support -- an unbounded general-purpose dataflow platform; the bounded governed BI - reference journey is selected +- unsupported breadth in native project management before the Projects/OpenProject + boundary is proved in a reference journey +- unbounded Dataflow operators or execution engines without golden-flow, + quality, lineage, resource-limit, and recovery evidence - every possible public-sector protocol adapter - rich LMS behavior beyond training administration - full qualified digital signing/trust services beyond the identity-trust and @@ -630,26 +649,26 @@ repositories or to explicit missing-module decisions. | Fully UI-managed configuration with safety controls | `govoplan-admin`, `govoplan-core`, `govoplan-policy`, `govoplan-access`, `govoplan-audit` | `GovOPlaN/govoplan-core#218` | | Access as a module | `govoplan-access` | `GovOPlaN/govoplan-access#7` | | Interface ethics and decision-consequence doctrine | `govoplan-core` plus all UI-owning modules | `GovOPlaN/govoplan-core#227` | -| Action/effect automation layer | first `govoplan-workflow`; possible future `govoplan-automation` | `GovOPlaN/govoplan-workflow#1` | +| Action/effect automation layer | `govoplan-workflow-engine`; possible future `govoplan-automation` only after boundary proof | `GovOPlaN/govoplan-workflow#1` | | E2EE role/function postbox architecture | `govoplan-postbox`, `govoplan-identity-trust`, `govoplan-access`, `govoplan-policy`, `govoplan-audit` | `GovOPlaN/govoplan-postbox#15`, `GovOPlaN/govoplan-identity-trust#1` | | Identity, account, function, role, right semantic model | `govoplan-access` | `GovOPlaN/govoplan-access#9` | -| Role-based service directory/catalog | `govoplan-portal` | `GovOPlaN/govoplan-portal#1` | +| Role-based service directory presentation | `govoplan-portal`; reusable service definition is a shared contract and candidate `govoplan-services` | `GovOPlaN/govoplan-portal#1` | | Unified inbox across tasks, postbox, notifications, and portal | `govoplan-core` coordination plus owning modules | `GovOPlaN/govoplan-tasks#1`, `GovOPlaN/govoplan-notifications#1` | | OpenProject API / project management connector | `govoplan-connectors` | `GovOPlaN/govoplan-connectors#1` | -| Native project-management module decision | connector-first through `govoplan-connectors`; no native project module yet | `GovOPlaN/govoplan-core#196`, `GovOPlaN/govoplan-connectors#1` | -| Datasources for databases, CSV, files, APIs | no repository yet; start with connectors/files/reporting and create `govoplan-datasources` only after the first package proves shared source-catalog ownership | `GovOPlaN/govoplan-core#197` | -| Dataflow for pipelines, BI, publication | no repository yet; start with workflow/reporting/connectors and create `govoplan-dataflow` only after repeated pipeline/lineage contracts emerge | `GovOPlaN/govoplan-core#198` | -| Monthly datasource and transformation workflows | first as configuration package across connectors, files, workflow, reporting, and templates | `GovOPlaN/govoplan-core#216` | +| Native project-management module | `govoplan-projects`; OpenProject transport remains connector-owned | `GovOPlaN/govoplan-projects#1`, `GovOPlaN/govoplan-connectors#12` | +| Datasources for databases, CSV, files, APIs | `govoplan-datasources`, with external protocol and credential ownership in Connectors | `GovOPlaN/govoplan-datasources#1` | +| Dataflow for pipelines, BI, publication | `govoplan-dataflow` over Datasources and module-owned providers | `GovOPlaN/govoplan-dataflow#1` | +| Monthly datasource and transformation workflows | configuration package across Connectors, Datasources, Dataflow, Workflow Engine, Reporting, Files, and Templates | `GovOPlaN/govoplan#8`, `GovOPlaN/govoplan-dataflow#17` | | Templates for letters, emails, forms, reports | `govoplan-templates`, separate from reporting | `GovOPlaN/govoplan-core#190`, `GovOPlaN/govoplan-templates#1` | | Reporting and BI | `govoplan-reporting`, separate from templates | `GovOPlaN/govoplan-core#190`, `GovOPlaN/govoplan-reporting#1` | | File connectors: Nextcloud, Seafile, SMB, NFS | `govoplan-files` | `GovOPlaN/govoplan-files#15` | | Public-sector software integration catalogue | `govoplan-connectors` with core strategy index | `GovOPlaN/govoplan-core#191`, `GovOPlaN/govoplan-connectors#2` | | Public-sector integration landscape catalogue | `govoplan-connectors` with core tracking | `GovOPlaN/govoplan-core#215` | | Cases module concept | `govoplan-cases` | `GovOPlaN/govoplan-core#174` | -| Workflow module concept | `govoplan-workflow` | `GovOPlaN/govoplan-core#175` | +| Workflow runtime/editor split | `govoplan-workflow-engine` runtime plus optional `govoplan-workflow` editor | `GovOPlaN/govoplan-workflow#12`, `GovOPlaN/govoplan-workflow#13` | | Connectors module concept | `govoplan-connectors` | `GovOPlaN/govoplan-core#176` | | Adrema-style address and distribution-list management | `govoplan-addresses` | `GovOPlaN/govoplan-addresses#1` | -| Consume sources and become a governed source | `govoplan-connectors` plus possible future `govoplan-dataflow` | `GovOPlaN/govoplan-connectors#3`, `GovOPlaN/govoplan-core#198` | +| Consume sources and become a governed source | Connectors acquires, Datasources governs/materializes, Dataflow transforms/publishes | `GovOPlaN/govoplan-connectors#3`, `GovOPlaN/govoplan-datasources#3` | | Governed connector configuration, dry-run, and simulation runtime | `govoplan-connectors` | `GovOPlaN/govoplan-connectors#6` | | Terminfindung and meeting scheduling polls | `govoplan-scheduling`; calendar primitives remain in calendar | `GovOPlaN/govoplan-core#193`, `GovOPlaN/govoplan-scheduling#1` | | Terminplaner and calendar primitives | `govoplan-calendar` | `GovOPlaN/govoplan-core#193`, `GovOPlaN/govoplan-calendar#1` | @@ -671,27 +690,23 @@ repositories or to explicit missing-module decisions. Boundary rationale lives in `MODULE_ARCHITECTURE.md`. Current decisions: - templates and reporting are separate modules -- RSS/source consume-publish starts in connectors; datasources/dataflow are not - repositories yet +- RSS/source consume-publish starts in Connectors; governed source identity and + snapshots belong to Datasources and transformations belong to Dataflow - calendar, scheduling, and appointments are three separate modules - forms definitions and forms runtime are separate responsibilities - OpenDesk is an integration profile across modules, not a monolithic module -- OpenProject is connector-first; no native projects module yet +- OpenProject transport is connector-owned; native portfolio/project semantics + belong to Projects - public-sector integration strategy stays in core; executable catalogue work lives in connectors - encrypted postbox and identity-trust are strategic contracts, not mail-module behavior -- automation starts as workflow-owned action/effect execution and may split into - a dedicated module only after the runner becomes broader than workflow - -The following modules are intentionally not created yet: - -- `govoplan-datasources` -- `govoplan-dataflow` -- `govoplan-projects` - -Create a repository only after a concrete implementation package proves that -existing connector, files, reporting, workflow, or task ownership is too narrow. +- automation starts in Workflow Engine and may split into a dedicated module + only after the runner becomes broader than workflow +- Mandates, Services, Parties, and Decisions begin as shared semantic contracts; + create repositories only after independent persistence, lifecycle, security, + and multiple-consumer evidence passes the repository threshold in the + institutional governance target architecture Core keeps the strategy index in `PUBLIC_SECTOR_INTEGRATION_STRATEGY.md`: integration postures, default diff --git a/docs/INSTITUTIONAL_CONTEXT_CONTRACT.md b/docs/INSTITUTIONAL_CONTEXT_CONTRACT.md new file mode 100644 index 0000000..8214eb0 --- /dev/null +++ b/docs/INSTITUTIONAL_CONTEXT_CONTRACT.md @@ -0,0 +1,159 @@ +# Institutional Context And Governed References + +GovOPlaN consequential work must retain enough context to answer who acted, +for whom, through which function, under which mandate and jurisdiction, using +which rule and evidence versions, and with which requested and observed effect. +The shared contract lives in `govoplan_core.core.institutional`. + +Core owns reference shapes and provider protocols only. It does not own shared +Mandate, Service, Party, Decision, evidence, or geography tables. Domain modules +own persistence and authorization; optional capabilities resolve the references. + +## Envelope + +`GovernedContextEnvelope` version 1 carries: + +- a tenant and `TemporalRevision` with validity, recording, supersession, and + change reason; +- the real account or service account and represented account, function, + procedure party, assignment, delegation/power, and mandate; +- institution, organization unit, function, task, mandate, jurisdiction, + service, case, party, work item, workflow, approval, decision, and record + references; +- versioned legal bases and evidence references; +- information classification, purposes, retention/holds, minimization, and + disclosure state; +- external-source authority, maturity, freshness, health, and conflict state; +- language, accessibility, channel, explanation, and availability references. + +Every institutional reference includes the owner module, tenant, stable object +identity, optional version/effective instant, and a protected display label. +Cross-tenant references are rejected. Safe serialization omits labels, +inspection URLs, formal reasoning, operative results, and conditions unless a +caller explicitly requests the protected projection. + +## Semantic Providers + +The first provider-neutral capabilities are: + +- `mandates.resolver`: resolve competence for a task/authority type at an + effective instant and return the governing Mandate definition and evidence; +- `services.definitions`: obtain versioned institutional service definitions; +- `parties.resolver`: obtain effective procedure-local parties and powers of + representation without copying Identity or Organizations subjects; and +- `decisions.registry`: record and retrieve formal Decisions under optimistic + revision control. + +The Mandate, Service, Party/representation, and Decision DTOs have strict +mapping round-trips so they can cross capability, event, package, and storage +boundaries without shared ORM models. Their lifecycle states are explicit: +Mandates distinguish draft/active/suspended/replaced/retired, Services retain +publication state, Parties retain effective representation and revocation, and +Decisions retain correction, revocation, and supersession references. + +The DTOs are a repository threshold, not a mandate to create four modules. +Independent persistence, lifecycle, security/operations behavior, release +reason, reuse, and tests are still required before extraction. + +Mandate resolution is deterministic: Core filters candidates by tenant, +effective interval, active state, task and authority type, stable +organization/function identity, jurisdiction coverage, and subject type. A +result is competent only when exactly one matching Mandate remains and it has +no unresolved conflicts. Evidence from matching definitions is deduplicated +and retained in the explanation result. `revise_mandate_definition` applies +optimistic concurrency and the allowed activation, suspension, replacement, +and retirement transitions while leaving the previous revision immutable. + +`revise_formal_decision` provides the equivalent lifecycle primitive for +formal outcomes. Every accepted transition requires a new recorded revision +and change reason, links `supersedes_ref` to the prior version, updates the +authority envelope to the new version, and records explicit correction or +revocation provenance. Terminal and backward transitions fail closed. Each +Decision also records whether responsibility was human, human-reviewed +automation, or an automated service account acting under mandate. Automation +preparation/recommendation references remain inspectable without being +mistaken for the responsible outcome. + +Procedure-party corrections use `revise_procedure_party`: the stable party +identity is retained, a new revision and reason are required, stale writes are +rejected, and revoked/expired/superseded assignments are terminal. +`revoke_party_representation` separately records when a limited power ceased +to authorize actions. This allows consuming procedures to evaluate historical +delivery or representation authority without rewriting Identity, +Organizations, or Addresses records. + +Service templates and package/tenant specializations use +`derive_service_restriction`. The derived definition retains an explicit +parent-version reference, cannot extend the parent's validity, audience, +channels, or publication ceiling, and cannot remove inherited prerequisites, +required evidence, legal bases, or bindings. This is the fail-closed semantic +rule; configuration-package signature and provenance checks remain the package +transport rule. + +`ServiceAvailabilityRequirement` represents module, capability, mandate, +policy, connector, maintenance, audience, and configuration prerequisites with +an explicit unavailable-or-hidden failure mode and explanation reference. The +optional `services.availability` evaluator returns policy-scoped boolean +assessments, reason codes, and evidence. Unknown consequential requirements +fail closed; a reference itself never grants access. + +## Service Launch + +`ServiceLaunchRequest` and `ServiceLaunchResult` define the owner-neutral +boundary between Portal entry and a case, form, or workflow runtime effect. +The request carries the exact published Service definition, exact selected +binding, tenant, acting identity, timezone-aware request time, bounded +parameters, and idempotency key. The result must retain that exact Service and +binding, a same-tenant target reference, optional same-tenant evidence, and +only a relative or credential-free HTTP(S) destination. + +`service_launch_capability(kind)` maps bindings to owner capabilities: + +- `case` -> `cases.service_launcher` +- `form` -> `forms_runtime.service_launcher` +- `workflow` -> `workflow_engine.service_launcher` + +`FormDefinition`, `FormFieldDefinition`, and `forms.definitions` provide the +owner-neutral exact-schema boundary used by Forms Runtime. Definitions carry an +exact tenant/revision, field types/options/constraints/defaults, publication, +draft, attachment, signature, policy, and handoff requirements. Forms owns +those immutable definitions; Forms Runtime persists instances and validation +evidence. A form Service binding uses `/` and the launcher +rejects missing, superseded, unpublished, cross-tenant, or invalid definitions. + +Portal may discover and invoke those capabilities but cannot write owner +tables. The owner must revalidate its definition/binding and current +authorization, produce its normal audit/event state, and make replay after an +ambiguous response safe. If the capability is absent, the service is +explainably unavailable. URL-only entries pass through the same launch-time +availability check and destination validation. Forms Runtime now supplies the +definition-aware form launcher when both Forms and Forms Runtime are active; +otherwise Portal continues to fail closed. + +## Propagation + +`PlatformEvent`, `ActionExecutionRequest`, and `AuditEvent` can carry the +envelope. Audit persistence stores only its safe projection; platform-event +outbox serialization preserves it across asynchronous delivery. A module must +not invent a parallel context dictionary when the shared fields apply. + +## First Proof + +Committee's `committee.decision_path` capability is the first bounded proof. It +requires one effective, conflict-free Mandate covering the organization unit +function, and jurisdiction, an approval reference, fact evidence, versioned legal bases, +operative result, and reasoning. It emits a reconstructable `FormalDecision`, +including requested/observed effects and information governance. If a Decision +registry is installed it persists there; Committee does not take ownership of +the generic Decision lifecycle. + +## Compatibility And Security + +- Contract version changes follow Core compatibility policy. +- Unknown tenant or reference-kind combinations fail closed. +- Datetimes that affect authority must be timezone-aware. +- Protected labels, reasoning, evidence inspection links, and source details + remain subject to the owning module's access policy. +- References do not grant access to their targets. +- Evidence and audit payloads must contain stable references/checksums, not + plaintext secrets. diff --git a/docs/MODULE_ARCHITECTURE.md b/docs/MODULE_ARCHITECTURE.md index 7a33c57..dc3f7fa 100644 --- a/docs/MODULE_ARCHITECTURE.md +++ b/docs/MODULE_ARCHITECTURE.md @@ -13,6 +13,10 @@ Policy decision, source provenance, and explain-response contracts are tracked in [`POLICY_CONTRACTS.md`](POLICY_CONTRACTS.md). The experimental remote WebUI bundle loading design is tracked in [`REMOTE_WEBUI_BUNDLES.md`](REMOTE_WEBUI_BUNDLES.md). +The cross-product semantic layers, source-authority modes, and candidate +Mandates, Services, Parties, and Decisions boundaries are canonical in the +meta repository's +[`INSTITUTIONAL_GOVERNANCE_TARGET_ARCHITECTURE.md`](../../govoplan/docs/INSTITUTIONAL_GOVERNANCE_TARGET_ARCHITECTURE.md). ## Layer Model @@ -24,6 +28,36 @@ The experimental remote WebUI bundle loading design is tracked in | Business modules | Public-sector workflows | campaigns, cases, forms, approvals, appointments | | Connector modules | External system integration | FIT-Connect, XÖV/XTA, DMS/eAkte, ERP, IDM | +This table is the technical composition model. The product portfolio uses a +more detailed institutional layer model, but it does not change dependency +direction: Core provides contracts and composition; modules own semantics; +packages compose modules. + +## Institutional Semantic Boundaries + +Cross-module references must keep these answers distinct: + +- Organizations owns where structures, units, and functions exist. +- Identity owns who a subject is; Access owns accounts, roles, permissions, + and authorization decisions; IDM owns effective function assignments. +- A Mandates capability will answer why a unit or function is competent for a + task, jurisdiction, subject, or period. It must not become another RBAC + system. +- A Services capability will own versioned institutional service definitions; + Portal presents and starts them. +- A Parties capability will own procedure-local participant roles, + representation, and delivery authority; it must reference rather than copy + Identity, Organizations, and Addresses subjects. +- A Decisions capability will own formal institutional outcomes and their + authority, facts, rules, reasoning, effects, correction, and review. + Approvals owns review gates, Committee owns deliberation/votes, and Workflow + Engine owns coordination. + +Start each missing concept as a versioned DTO/provider contract used by a +bounded journey. A repository is justified only when the concept gains +independent persistence, lifecycle, security/operations behavior, release +reason, and reuse. Core must not store these domain objects. + ## Kernel Responsibilities The kernel target owns: @@ -98,6 +132,11 @@ The following contracts are the baseline API that modules can rely on: - navigation metadata contract - command/event envelope contract - policy decision and source provenance contract in `govoplan_core.core.policy` +- external object reference and integration-maturity contract in + `govoplan_core.core.external_references` +- action/effect preview and execution contract in + `govoplan_core.core.automation` +- workflow definition contribution and runtime-worker contracts Changes to these contracts must be versioned or accompanied by compatibility shims. @@ -115,6 +154,30 @@ may extend the kernel by adding explicit contracts, but existing contracts must remain source-compatible through the 0.1.x split line unless a migration shim and deprecation note are provided. +### Architecture Metadata + +`ModuleManifest.architecture` is the backward-compatible, versioned product- +portfolio declaration for: + +- module kind and institutional architecture layer; +- evidence-backed maturity claim (`concept`, `scaffold`, `vertical_slice`, + `reference_ready`, `supported`, or `lts`); +- owned and explicitly non-owned concepts; +- supported source-authority modes; +- reference packages, tested providers, and known limits; +- migration, upgrade, recovery, security, operations, and documentation + evidence references. + +Core validates the claim and all provider references during registry startup. +`reference_ready`, `supported`, and `lts` claims require a named reference +package and the cumulative evidence set; target-tested providers additionally +require provider evidence. A supported module with migrations must include +migration evidence. Signed release catalogs retain and revalidate the +declaration. The meta manifest check validates repository evidence paths and +provides an opt-in `--require-architecture` rollout gate. Platform metadata, +Docs, and Ops project the same declaration. A module cannot make itself +supported solely by changing its maturity string. + Known access-related capability names are defined in `govoplan_core.core.access`, including: @@ -180,6 +243,45 @@ intended for SemVer major-version lines. Missing optional interfaces are allowed, but an installed provider with an incompatible version blocks activation because the integration would otherwise bind to an unsafe API. +### Source Authority And Provider Operations + +Integration maturity and configured authority are independent. The existing +external-reference maturity ladder describes whether an adapter can discover, +link, search, read, publish, synchronize, migrate, or replace. A binding must +also state whether GovOPlaN is native authoritative, the external system is +authoritative, GovOPlaN keeps a mirror, both sides use governed sync, GovOPlaN +adds only a governance overlay, or the object is link-only. + +`ModuleManifest.external_providers` composes existing contracts rather than +replacing them. Each declaration describes owned object/field groups, authority modes, +operations, revisions, freshness, health, limits, idempotency, conflicts, +outcome-unknown handling, evidence, correction/compensation, reconciliation, +outage behavior, classification, purpose, retention, and secret requirements. +Core owns the typed declaration and validation. Connectors and domain modules +own the actual protocol and domain behavior; configuration packages select the +effective mode and block missing/incompatible/unhealthy providers; Docs and Ops +explain the result. Effect-capable declarations fail validation unless their +retry, concurrency, outcome-unknown, correction, reconciliation, evidence, +audit, timeout, outage, classification, purpose, retention, and secret behavior +is explicit. + +Declarations are release-time capability claims. Configured state is projected +separately through `ModuleManifest.external_provider_state_providers`. A state +provider receives a bounded tenant context and returns one sanitized observation +per configured binding: stable binding reference, effective authority mode, +active/configured state, health, freshness, conflict, recovery readiness, +observation/last-success time, and scalar metrics. Core validates and aggregates +those observations, isolates provider failures, and never accepts URLs, +credentials, or arbitrary nested metadata as runtime-state fields. Docs removes +binding-level detail from ordinary-user projections; Ops may show the full +sanitized operator projection. + +Configuration-package preflight selects the exact requested binding from this +runtime state before evaluating authority, health, freshness, and recovery. A +healthy sibling binding therefore cannot mask an unhealthy required binding. +Providers with multiple configurations must use non-secret, stable references +such as `calendar:sync-source:`. + Current named interfaces, generated from the source manifests by the workspace contract checks, are: @@ -593,6 +695,39 @@ Rules: one migration run, and do not switch a database between tracks unless it is a disposable development database. +### Shared State And Runtime Ordering + +Multi-host application roles use the `shared` state profile. In that profile, +PostgreSQL, Redis, a stable installation identifier, and S3-compatible object +storage are mandatory. Module durable artifacts must use Core's object-storage +contract and module-owned opaque key namespaces; node-local paths are limited +to temporary materialization. Same-host replicas may use the `host-shared` +profile and one shared volume. + +Only the migration command mutates schema. PostgreSQL migration runs acquire a +deployment-wide advisory lock before module pre-tasks, Alembic, and post-tasks. +API, worker, and scheduler roles wait for exact configured migration heads and +fail closed instead of applying migrations during startup. + +Runtime roles register identity, software/module composition, queues, heartbeat, +and drain state in PostgreSQL. Singleton work must use a distributed lease and +validate its monotonically increasing fencing token at the consequential +commit. See `STATE_AND_RECOVERY_CONTRACT.md` for the complete contract. + +### Recovery Evidence + +Operations spanning transactions, object storage, queues, or external systems +must choose an explicit Core recovery mode: atomic, compensation, +snapshot-restore, forward-recovery, or irreversible. Plans require verification +steps and mode-specific recovery material. Use idempotency keys, append-only +evidence checkpoints, and a runtime fence where work may race across nodes. + +The recovery ledger is a shared primitive, not automatic coverage. A module may +claim its guarantees only after its operation records preconditions before side +effects, transitions partial/unknown outcomes honestly, and records verified +completion or recovery. Plaintext secrets must never enter recovery metadata or +evidence. + ## Install, Uninstall, And Catalogs Core owns the install plan, signed catalog validation, license entitlement @@ -962,7 +1097,7 @@ from workflow semantics. - form definitions, schemas, validation rules, field visibility rules, localization, versioning, admin editing, and reusable form package fragments -`govoplan-forms-runtime` owns, when implemented: +`govoplan-forms-runtime` owns: - public/internal submissions, drafts, submitted values, validation evidence, attachment references, submission receipts, and handoff events @@ -975,6 +1110,16 @@ Boundary: - Reporting/dataflow may consume submitted data through governed DTOs or source lifecycle contracts. +Implemented contract: + +- Core owns the provider-neutral `FormDefinition`/`FormFieldDefinition` DTOs. +- Forms persists immutable exact definitions and provides `forms.definitions`. +- Forms Runtime resolves that capability, persists revisioned instances and + events, validates draft/final values, and provides + `forms_runtime.service_launcher`. +- Portal delegates exact `/` bindings and never writes either + owner's tables. + ### OpenDesk Integration Profile Tracking: `govoplan-core#195`, `govoplan-connectors#5`, @@ -1219,6 +1364,9 @@ the same restart/health set after restoring package and database snapshots. The installer preflight is intentionally conservative: - maintenance mode must be active; +- the `shared` state profile blocks in-place package mutation; clustered + installations must roll one verified immutable module composition across all + replicas; - installed module manifests must be compatible with the supported manifest contract and current core version; - uninstalling `tenancy`, `access`, or `admin` is blocked; diff --git a/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md b/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md index 8cc2b6f..7a0adad 100644 --- a/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md +++ b/docs/PUBLIC_SECTOR_INTEGRATION_STRATEGY.md @@ -175,8 +175,10 @@ connector or module issue. queries, untraceable manual transformations. - MVP test path: publish one report/export as a governed file plus RSS/Atom entry with checksum, timestamp, and permission check. -- Owner/priority: `govoplan-reporting`, `govoplan-connectors`, possible future - `govoplan-datasources`/`govoplan-dataflow`, Wave 2. +- Owner/priority: `govoplan-reporting`, `govoplan-connectors`, + `govoplan-datasources`, and `govoplan-dataflow`, Wave 2. Reporting owns + presentation/publication, Connectors owns transport, Datasources owns the + governed source/materialization catalogue, and Dataflow owns transformations. ### Public-Sector Protocols And Registries diff --git a/docs/STATE_AND_RECOVERY_CONTRACT.md b/docs/STATE_AND_RECOVERY_CONTRACT.md new file mode 100644 index 0000000..b99ab67 --- /dev/null +++ b/docs/STATE_AND_RECOVERY_CONTRACT.md @@ -0,0 +1,143 @@ +# State And Recovery Contract + +## State Profiles + +Core accepts three runtime state profiles: + +| Profile | Runtime placement | Durable storage | +| --- | --- | --- | +| `local` | One development process set | Local filesystem is permitted. | +| `host-shared` | Replicas on one host | One host-shared volume plus PostgreSQL and Redis. | +| `shared` | Independent hosts | PostgreSQL, Redis, and S3-compatible object storage. | + +All replicas in one installation use one stable +`GOVOPLAN_INSTALLATION_ID`, one immutable module composition, and identical +database, broker, encryption-key, and object-storage bindings. Core rejects +replicas with the `local` profile and rejects `shared` without PostgreSQL, +Redis, S3, and a non-default installation identifier. + +`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true` is an explicit deployment trust +declaration for a clean HTTPS S3 origin. It does not authorize a +user-controlled connector endpoint and it is separate from installer-managed +Garage's exact endpoint trust. + +## Object Storage + +`govoplan_core.core.object_storage` is the shared backend contract for durable +module artifacts. It provides bounded read/write/list/stat/delete operations +for local and S3-compatible storage. Modules own their object-key namespace and +business metadata; Core does not interpret module files. + +Rules for modules: + +- Store only opaque object keys in business records, never local absolute + paths. +- Use node-local directories only for temporary materialization. +- Verify expected size and digest before consuming consequential artifacts. +- If object creation precedes database commit, compensate successfully created + objects on failure. +- If object deletion fails, retain the database reference and report a retryable + failure rather than claiming deletion. +- Define an orphan-inventory strategy for hard process loss between object + creation and metadata commit. + +The Files module delegates its backend implementation to this Core contract. +Campaign generated EML artifacts use a Campaign-owned object prefix and are +read by workers through the same shared backend. + +## Runtime Nodes And Leases + +API and worker incarnations register in `core_runtime_nodes` with role, +software version, module-composition hash, queues, start time, and heartbeat. +The registration identity includes a process incarnation so a stale process +cannot update a replacement's row. + +Drain is durable operator intent: + +- an API enters not-ready state after observing drain; +- a worker cancels queue consumers after observing drain; +- cancellation returns an eligible draining node to active state; +- clean shutdown marks the matching incarnation stopped. + +Coordination loss also fails closed. An API reports +`coordination_unavailable` from `/health/ready` until a heartbeat succeeds +again. A worker cancels its local queue consumers on any heartbeat or database +failure and only resumes them after its existing incarnation heartbeats +successfully. It never re-registers from the heartbeat path, so a stale worker +cannot reclaim a node identity from its replacement. + +`core_distributed_leases` provides installation/resource uniqueness, expiry, +holder incarnation, and monotonically increasing fencing tokens. Lease expiry +does not make an old process harmless by itself: code performing an effect must +assert its exact fence before commit. `govoplan_core.commands.fenced_run` +renews a lease around a subprocess and terminates the child when the lease is +lost. The deployment profiles use it for the singleton scheduler. + +## Migration Ordering + +Only `govoplan_core.commands.init_db` mutates schema. On PostgreSQL it acquires +a deterministic installation/track advisory lock before pre-migration tasks, +Alembic, and post-migration tasks. The lock is session-scoped and therefore +released if the migration process dies. + +Runtime roles run `govoplan_core.commands.wait_for_database`. It waits until +the database has exactly the configured Core/module Alembic heads and never +upgrades schema. This permits a migration Job and runtime Deployments to be +submitted together while keeping startup fail-closed. + +## Recovery Ledger + +`govoplan_core.core.recovery` provides a durable operation and evidence +contract. Recovery modes are: + +- `atomic`: one database transaction, no external effect; +- `compensation`: explicit inverse actions; +- `snapshot_restore`: separately verified backup reference; +- `forward_recovery`: repair/resume the current version; +- `irreversible`: explicit approval, no automated recovery claim. + +Every plan requires verification steps. Mode-specific evidence is mandatory. +Operations bind an idempotency key to a canonical request hash, may bind a +runtime fencing token, and emit append-only hash-chained checkpoints. Backup +and approval references are part of the hashed plan evidence. Every low-level +state transition and checkpoint append revalidates the operation's recorded +fence while holding the operation row lock. Plaintext secrets are rejected +from metadata and evidence. + +The state machine makes partial and uncertain outcomes visible. A non-atomic +running operation cannot transition directly to ordinary failure, and success +or recovery requires explicit verified checks. Ops projects states requiring +attention, but module behavior gains this guarantee only after it adopts the +ledger around its own side effects. + +## Recovery Boundary + +Application/configuration rollback and database rollback are not equivalent. +Once an incompatible migration starts, old code may be unsafe even if its image +is available. Deployment automation must switch to forward recovery unless a +coordinated and verified database/object/key backup is restored. + +Core does not create production database backups. The deployment owner must +provide backup, retention, encryption, restore verification, and recovery-point +coordination for PostgreSQL, object storage, and encryption keys. The canonical +operator procedure is documented in +`govoplan/docs/RECOVERY_AND_ROLLBACK_GUARANTEES.md`. + +## Verification + +Focused contracts are covered by: + +```sh +/mnt/DATA/git/govoplan/.venv/bin/python -m pytest -q \ + tests/test_object_storage.py \ + tests/test_runtime_coordination.py \ + tests/test_runtime_agents.py \ + tests/test_fenced_run.py \ + tests/test_migration_lock.py \ + tests/test_wait_for_database.py \ + tests/test_recovery_guarantees.py +``` + +Production acceptance additionally requires multi-node failure and coordinated +restore drills against the actual PostgreSQL, Redis, object-store, ingress, and +secret-provider topology. diff --git a/pyproject.toml b/pyproject.toml index a327e9c..f6d5f21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "celery>=5,<6", "redis>=5,<6", "alembic>=1,<2", + "boto3>=1.34,<2", ] [tool.setuptools.packages.find] diff --git a/src/govoplan_core/audit/logging.py b/src/govoplan_core/audit/logging.py index de1b0ff..e936f33 100644 --- a/src/govoplan_core/audit/logging.py +++ b/src/govoplan_core/audit/logging.py @@ -18,6 +18,7 @@ from govoplan_core.core.events import ( normalize_trace_id, emit_platform_event, ) +from govoplan_core.core.institutional import GovernedContextEnvelope from govoplan_core.core.runtime import get_registry from govoplan_core.privacy.retention import sanitize_audit_details_for_policy from govoplan_core.security.redaction import redact_secret_values @@ -176,6 +177,16 @@ def _publish_audit_platform_event( item: AuditRecordRef, ) -> None: trace = _compact_trace(item.details.get("_trace") if isinstance(item.details, Mapping) else None) + raw_context = ( + item.details.get("_institutional_context") + if isinstance(item.details, Mapping) + else None + ) + institutional_context = ( + GovernedContextEnvelope.from_mapping(raw_context) + if isinstance(raw_context, Mapping) + else None + ) emit_platform_event( session, PlatformEvent( @@ -194,6 +205,7 @@ def _publish_audit_platform_event( tenant=EventTenantRef(id=item.tenant_id) if item.tenant_id else None, resource=EventObjectRef(type=item.object_type, id=item.object_id) if item.object_type else None, classification="internal", + institutional_context=institutional_context, ) ) @@ -211,6 +223,7 @@ def audit_event( details: dict[str, Any] | None = None, correlation_id: str | None = None, causation_id: str | None = None, + institutional_context: GovernedContextEnvelope | None = None, commit: bool = False, ) -> AuditRecordRef: """Persist one audit event. @@ -223,8 +236,17 @@ def audit_event( if scope not in {"tenant", "system"}: raise ValueError(f"Unsupported audit scope: {scope}") + if ( + institutional_context is not None + and tenant_id is not None + and institutional_context.tenant_id != tenant_id + ): + raise ValueError("Audit institutional context belongs to another tenant") + raw_details = dict(details or {}) + if institutional_context is not None: + raw_details["_institutional_context"] = institutional_context.to_dict() traced_details, trace = _trace_details( - _sanitize_details(details or {}), + _sanitize_details(raw_details), correlation_id=correlation_id, causation_id=causation_id, ) @@ -242,6 +264,7 @@ def audit_event( api_key_id=api_key_id, resource_type=object_type, resource_id=object_id, + institutional_context=institutional_context, details=stored_details, )) record_change( @@ -273,6 +296,7 @@ def audit_from_principal( details: dict[str, Any] | None = None, correlation_id: str | None = None, causation_id: str | None = None, + institutional_context: GovernedContextEnvelope | None = None, commit: bool = False, ) -> AuditRecordRef: return audit_event( @@ -287,5 +311,6 @@ def audit_from_principal( details=details, correlation_id=correlation_id, causation_id=causation_id, + institutional_context=institutional_context, commit=commit, ) diff --git a/src/govoplan_core/celery_app.py b/src/govoplan_core/celery_app.py index a456154..49e28b2 100644 --- a/src/govoplan_core/celery_app.py +++ b/src/govoplan_core/celery_app.py @@ -1,11 +1,20 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone +from importlib.metadata import PackageNotFoundError, version +import logging from celery import Celery +from celery.signals import heartbeat_sent, worker_ready, worker_shutdown -from govoplan_core.core.campaigns import CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, CampaignDeliveryTaskProvider -from govoplan_core.core.calendar import CAPABILITY_CALENDAR_OUTBOX, CalendarOutboxProvider +from govoplan_core.core.campaigns import ( + CAPABILITY_CAMPAIGNS_DELIVERY_TASKS, + CampaignDeliveryTaskProvider, +) +from govoplan_core.core.calendar import ( + CAPABILITY_CALENDAR_OUTBOX, + CalendarOutboxProvider, +) from govoplan_core.core.dataflows import ( CAPABILITY_DATAFLOW_RUN_WORKER, CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER, @@ -23,7 +32,10 @@ from govoplan_core.core.idm import ( CAPABILITY_IDM_ASSIGNMENT_LIFECYCLE, IdmAssignmentLifecycle, ) -from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids +from govoplan_core.core.module_management import ( + load_startup_enabled_modules, + startup_candidate_module_ids, +) from govoplan_core.core.mail import ( CAPABILITY_MAIL_BOUNCE_PROCESSING, CAPABILITY_MAIL_DELIVERY_OUTBOX, @@ -31,7 +43,10 @@ from govoplan_core.core.mail import ( MailDeliveryOutboxProvider, ) from govoplan_core.core.modules import ModuleContext -from govoplan_core.core.notifications import CAPABILITY_NOTIFICATIONS_DISPATCH, NotificationDispatchProvider +from govoplan_core.core.notifications import ( + CAPABILITY_NOTIFICATIONS_DISPATCH, + NotificationDispatchProvider, +) from govoplan_core.core.postbox import ( CAPABILITY_POSTBOX_ROUTING, PostboxRoutingProvider, @@ -42,9 +57,19 @@ from govoplan_core.core.workflows import ( ) from govoplan_core.core.registry import PlatformRegistry from govoplan_core.core.runtime import configure_runtime +from govoplan_core.core.runtime_coordination import ( + RuntimeIdentity, + heartbeat_runtime_node, + register_runtime_node, + runtime_identity, + stop_runtime_node, +) from govoplan_core.settings import settings -from govoplan_core.db.session import configure_database -from govoplan_core.server.registry import available_module_manifests, build_platform_registry +from govoplan_core.db.session import configure_database, get_database +from govoplan_core.server.registry import ( + available_module_manifests, + build_platform_registry, +) configure_database(settings.database_url) @@ -141,6 +166,119 @@ celery.conf.update( }, ) +_worker_identity: RuntimeIdentity | None = None +_worker_draining = False +_worker_consumer: object | None = None +logger = logging.getLogger("govoplan.worker.runtime") + + +def _core_version() -> str: + try: + return version("govoplan-core") + except PackageNotFoundError: + return "development" + + +def _worker_runtime_identity(sender: object | None = None) -> RuntimeIdentity: + global _worker_identity + if _worker_identity is None: + hostname = str(getattr(sender, "hostname", "") or "").strip() or None + module_ids = tuple(load_startup_enabled_modules(settings.enabled_modules)) + _worker_identity = runtime_identity( + settings, + software_version=_core_version(), + module_ids=module_ids, + role="worker", + node_id=hostname, + ) + return _worker_identity + + +@worker_ready.connect +def _register_worker_runtime(sender=None, **_kwargs) -> None: + global _worker_consumer + _worker_consumer = sender + identity = _worker_runtime_identity(sender) + try: + with get_database().SessionLocal() as session: + node = register_runtime_node( + session, + identity, + metadata={"process": "celery-worker"}, + ) + session.commit() + except Exception: + _set_worker_consumers(identity, draining=True) + logger.exception( + "worker runtime registration failed; queues were disabled node_id=%s", + identity.node_id, + ) + raise + _set_worker_consumers(identity, draining=node.state == "draining") + + +@heartbeat_sent.connect +def _heartbeat_worker_runtime(sender=None, **_kwargs) -> None: + identity = _worker_runtime_identity(sender) + try: + with get_database().SessionLocal() as session: + node = heartbeat_runtime_node( + session, + identity, + metadata={"process": "celery-worker"}, + ) + session.commit() + except Exception: # noqa: BLE001 - uncertain authority must fail closed + _set_worker_consumers(identity, draining=True) + logger.exception( + "worker runtime heartbeat failed; queues were disabled node_id=%s", + identity.node_id, + ) + return + _set_worker_consumers(identity, draining=node.state == "draining") + + +def _set_worker_consumers( + identity: RuntimeIdentity, + *, + draining: bool, +) -> None: + global _worker_draining + if draining == _worker_draining: + return + consumer = _worker_consumer + if consumer is not None: + method_name = "cancel_task_queue" if draining else "add_task_queue" + method = getattr(consumer, method_name) + destination = None + else: + method = ( + celery.control.cancel_consumer if draining else celery.control.add_consumer + ) + destination = [identity.node_id] + for queue in identity.queues: + if destination is None: + method(queue) + else: + method(queue, destination=destination) + _worker_draining = draining + + +@worker_shutdown.connect +def _stop_worker_runtime(sender=None, **_kwargs) -> None: + identity = _worker_identity + if identity is None: + return + try: + with get_database().SessionLocal() as session: + stop_runtime_node(session, identity) + session.commit() + except Exception: # noqa: BLE001 - shutdown must continue + logger.exception( + "worker runtime stop marker failed node_id=%s", + identity.node_id, + ) + @celery.task(name="govoplan.ping") def ping(): @@ -149,9 +287,15 @@ def ping(): def _platform_registry() -> PlatformRegistry: raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules) - candidate_modules = startup_candidate_module_ids(settings.enabled_modules, raw_enabled_modules) - available_modules = available_module_manifests(enabled_modules=candidate_modules, ignore_load_errors=True) - enabled_modules = load_startup_enabled_modules(settings.enabled_modules, available=available_modules) + candidate_modules = startup_candidate_module_ids( + settings.enabled_modules, raw_enabled_modules + ) + available_modules = available_module_manifests( + enabled_modules=candidate_modules, ignore_load_errors=True + ) + enabled_modules = load_startup_enabled_modules( + settings.enabled_modules, available=available_modules + ) registry = build_platform_registry(enabled_modules) context = ModuleContext(registry=registry, settings=settings) configure_runtime(context) @@ -211,9 +355,7 @@ def _dataflow_trigger_dispatcher( registry = registry or _platform_registry() if not registry.has_capability(CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER): return None - capability = registry.require_capability( - CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER - ) + capability = registry.require_capability(CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER) if not isinstance(capability, DataflowTriggerDispatcher): raise RuntimeError("Dataflow trigger dispatcher capability is invalid") return capability @@ -237,9 +379,7 @@ def _workflow_runtime_worker( registry = registry or _platform_registry() if not registry.has_capability(CAPABILITY_WORKFLOW_RUNTIME_WORKER): return None - capability = registry.require_capability( - CAPABILITY_WORKFLOW_RUNTIME_WORKER - ) + capability = registry.require_capability(CAPABILITY_WORKFLOW_RUNTIME_WORKER) if not isinstance(capability, WorkflowRuntimeWorker): raise RuntimeError("Workflow runtime worker capability is invalid") return capability @@ -275,9 +415,7 @@ def _idm_assignment_lifecycle( registry = registry or _platform_registry() if not registry.has_capability(CAPABILITY_IDM_ASSIGNMENT_LIFECYCLE): return None - capability = registry.require_capability( - CAPABILITY_IDM_ASSIGNMENT_LIFECYCLE - ) + capability = registry.require_capability(CAPABILITY_IDM_ASSIGNMENT_LIFECYCLE) if not isinstance(capability, IdmAssignmentLifecycle): raise RuntimeError("IDM assignment lifecycle capability is invalid") return capability @@ -295,7 +433,11 @@ def send_email(self, job_id: str): from govoplan_core.db.session import get_database with get_database().SessionLocal() as session: - return dict(_campaign_delivery_tasks().send_campaign_job(session, job_id=job_id, enqueue_imap_task=True)) + return dict( + _campaign_delivery_tasks().send_campaign_job( + session, job_id=job_id, enqueue_imap_task=True + ) + ) @celery.task(name="govoplan.campaigns.append_sent", bind=True, max_retries=None) @@ -306,7 +448,9 @@ def append_sent(self, job_id: str): with get_database().SessionLocal() as session: try: - return dict(_campaign_delivery_tasks().append_sent_for_job(session, job_id=job_id)) + return dict( + _campaign_delivery_tasks().append_sent_for_job(session, job_id=job_id) + ) except Exception as exc: if getattr(exc, "temporary", None) is True: raise self.retry(exc=exc, countdown=300) @@ -318,7 +462,11 @@ def deliver_notification(self, notification_id: str): from govoplan_core.db.session import get_database with get_database().SessionLocal() as session: - result = dict(_notification_dispatch().deliver_notification(session, notification_id=notification_id)) + result = dict( + _notification_dispatch().deliver_notification( + session, notification_id=notification_id + ) + ) session.commit() return result @@ -328,7 +476,11 @@ def deliver_pending_notifications(self, tenant_id: str | None = None, limit: int from govoplan_core.db.session import get_database with get_database().SessionLocal() as session: - result = dict(_notification_dispatch().deliver_pending(session, tenant_id=tenant_id, limit=limit)) + result = dict( + _notification_dispatch().deliver_pending( + session, tenant_id=tenant_id, limit=limit + ) + ) session.commit() return result @@ -411,7 +563,13 @@ def dispatch_calendar_outbox(self, tenant_id: str | None = None, limit: int = 50 with get_database().SessionLocal() as session: provider = _calendar_outbox() if provider is None: - return {"processed": 0, "succeeded": 0, "retrying": 0, "failed": 0, "operations": []} + return { + "processed": 0, + "succeeded": 0, + "retrying": 0, + "failed": 0, + "operations": [], + } result = dict(provider.dispatch_due(session, tenant_id=tenant_id, limit=limit)) session.commit() return result @@ -608,6 +766,7 @@ def dispatch_platform_events(self, limit: int = 100): dataflow_dispatcher = _dataflow_trigger_dispatcher(registry) consumers = () if dataflow_dispatcher is not None: + def deliver_to_dataflow( event: PlatformEvent, _delivery_key: str, diff --git a/src/govoplan_core/commands/fenced_run.py b/src/govoplan_core/commands/fenced_run.py new file mode 100644 index 0000000..cf0a0b7 --- /dev/null +++ b/src/govoplan_core/commands/fenced_run.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import argparse +from importlib.metadata import PackageNotFoundError, version +import signal +import subprocess +import sys +import threading +import time +from typing import Sequence + +from govoplan_core.core.runtime_coordination import ( + LeaseClaim, + acquire_lease, + release_lease, + renew_lease, + runtime_identity, +) +from govoplan_core.db.session import configure_database, get_database +from govoplan_core.settings import settings + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run one process while holding a database-fenced deployment lease." + ) + parser.add_argument( + "--resource", required=True, help="Stable cluster-wide lease key." + ) + parser.add_argument("--ttl-seconds", type=int, default=60) + parser.add_argument("--renew-seconds", type=int, default=15) + parser.add_argument("--wait-seconds", type=int, default=0) + parser.add_argument("command", nargs=argparse.REMAINDER) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + command = list(args.command) + if command and command[0] == "--": + command.pop(0) + if not command: + raise SystemExit("fenced-run requires a command after --") + if args.ttl_seconds < 10: + raise SystemExit("--ttl-seconds must be at least 10") + if args.renew_seconds < 2 or args.renew_seconds * 2 >= args.ttl_seconds: + raise SystemExit("--renew-seconds must be less than half the lease TTL") + configure_database(settings.database_url) + identity = runtime_identity( + settings, + software_version=_core_version(), + role=str(settings.runtime_role or "deployment"), + ) + claim = _wait_for_lease( + resource=args.resource, + identity=identity, + ttl_seconds=args.ttl_seconds, + wait_seconds=max(0, args.wait_seconds), + ) + if claim is None: + print( + f"lease unavailable: {args.resource}", + file=sys.stderr, + ) + return 75 + + process = subprocess.Popen(command) # noqa: S603 - argv is an operator-owned container command + stop = threading.Event() + fence_lost = threading.Event() + renewer = threading.Thread( + target=_renew_loop, + kwargs={ + "claim": claim, + "ttl_seconds": args.ttl_seconds, + "renew_seconds": args.renew_seconds, + "stop": stop, + "fence_lost": fence_lost, + "process": process, + }, + daemon=True, + name=f"govoplan-fence:{args.resource}", + ) + renewer.start() + previous_handlers = _forward_signals(process) + try: + return_code = process.wait() + finally: + stop.set() + renewer.join(timeout=args.renew_seconds + 2) + _restore_signals(previous_handlers) + _release(claim) + if fence_lost.is_set(): + return 74 + return int(return_code) + + +def _wait_for_lease( + *, + resource: str, + identity, + ttl_seconds: int, + wait_seconds: int, +) -> LeaseClaim | None: + deadline = time.monotonic() + wait_seconds + while True: + with get_database().SessionLocal() as session: + claim = acquire_lease( + session, + installation_id=identity.installation_id, + resource_key=resource, + holder_node_id=identity.node_id, + holder_incarnation=identity.incarnation, + ttl_seconds=ttl_seconds, + metadata={"role": identity.role}, + ) + session.commit() + if claim is not None or time.monotonic() >= deadline: + return claim + time.sleep(min(2, max(0.1, deadline - time.monotonic()))) + + +def _renew_loop( + *, + claim: LeaseClaim, + ttl_seconds: int, + renew_seconds: int, + stop: threading.Event, + fence_lost: threading.Event, + process: subprocess.Popen[bytes], +) -> None: + active_claim = claim + while not stop.wait(renew_seconds): + try: + with get_database().SessionLocal() as session: + active_claim = renew_lease( + session, + active_claim, + ttl_seconds=ttl_seconds, + ) + session.commit() + except Exception: # noqa: BLE001 - any renewal failure loses authority + fence_lost.set() + _terminate_process(process) + return + + +def _terminate_process( + process: subprocess.Popen[bytes], + *, + timeout_seconds: float = 5.0, +) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=timeout_seconds) + + +def _release(claim: LeaseClaim) -> None: + try: + with get_database().SessionLocal() as session: + release_lease(session, claim) + session.commit() + except Exception: # noqa: BLE001 - authority is already lost; release is best effort + return + + +def _forward_signals( + process: subprocess.Popen[bytes], +) -> dict[int, signal.Handlers]: + previous: dict[int, signal.Handlers] = {} + + def forward(signum, _frame) -> None: + if process.poll() is None: + process.send_signal(signum) + + for signum in (signal.SIGTERM, signal.SIGINT): + previous[signum] = signal.getsignal(signum) + signal.signal(signum, forward) + return previous + + +def _restore_signals(previous: dict[int, signal.Handlers]) -> None: + for signum, handler in previous.items(): + signal.signal(signum, handler) + + +def _core_version() -> str: + try: + return version("govoplan-core") + except PackageNotFoundError: + return "development" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/govoplan_core/commands/init_db.py b/src/govoplan_core/commands/init_db.py index e207e85..e5a68d6 100644 --- a/src/govoplan_core/commands/init_db.py +++ b/src/govoplan_core/commands/init_db.py @@ -12,6 +12,7 @@ from govoplan_core.db.migrations import ( migrate_database, run_registered_module_migration_tasks, ) +from govoplan_core.db.migration_lock import deployment_migration_lock from govoplan_core.db.session import configure_database, get_database from govoplan_core.settings import settings @@ -28,6 +29,12 @@ def main() -> None: parser.add_argument("--enabled-module", action="append", default=[], help="Target enabled module id used to discover module migrations; may be repeated.") parser.add_argument("--migration-module", action="append", default=[], help="Module id whose migration heads should be upgraded in this order before final heads.") parser.add_argument("--migration-task-record-output", type=Path, help="Write executed module migration task records to this JSON file.") + parser.add_argument( + "--migration-lock-timeout-seconds", + type=float, + default=900.0, + help="Maximum wait for the deployment-wide PostgreSQL advisory lock.", + ) parser.add_argument("--with-dev-data", action="store_true", help="Create default tenant/user/roles and a development API key") parser.add_argument("--dev-api-key", default=settings.dev_bootstrap_api_key, help="Development API key secret to create") args = parser.parse_args() @@ -37,26 +44,32 @@ def main() -> None: migration_order = tuple(args.migration_module) if args.migration_module else None task_records: list[dict[str, object]] = [] try: - _run_migration_tasks( - task_records, - database_url=args.database_url, - enabled_modules=enabled_modules, - migration_order=migration_order, - phases=PRE_MIGRATION_TASK_PHASES, - ) - migration = migrate_database( - database_url=args.database_url, - enabled_modules=enabled_modules, - migration_module_order=migration_order, + with deployment_migration_lock( + args.database_url, + installation_id=settings.installation_id, migration_track=args.migration_track, - ) - _run_migration_tasks( - task_records, - database_url=args.database_url, - enabled_modules=enabled_modules, - migration_order=migration_order, - phases=POST_MIGRATION_TASK_PHASES, - ) + timeout_seconds=args.migration_lock_timeout_seconds, + ): + _run_migration_tasks( + task_records, + database_url=args.database_url, + enabled_modules=enabled_modules, + migration_order=migration_order, + phases=PRE_MIGRATION_TASK_PHASES, + ) + migration = migrate_database( + database_url=args.database_url, + enabled_modules=enabled_modules, + migration_module_order=migration_order, + migration_track=args.migration_track, + ) + _run_migration_tasks( + task_records, + database_url=args.database_url, + enabled_modules=enabled_modules, + migration_order=migration_order, + phases=POST_MIGRATION_TASK_PHASES, + ) finally: if args.migration_task_record_output: args.migration_task_record_output.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/govoplan_core/commands/wait_for_database.py b/src/govoplan_core/commands/wait_for_database.py new file mode 100644 index 0000000..6b511d8 --- /dev/null +++ b/src/govoplan_core/commands/wait_for_database.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import sys +import time + +from govoplan_core.db.migrations import ( + configured_migration_heads, + database_migration_heads, +) +from govoplan_core.settings import settings + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Wait until the database is at this release's migration heads." + ) + parser.add_argument("--database-url", default=settings.database_url) + parser.add_argument( + "--migration-track", + default=settings.migration_track, + choices=("release", "dev"), + ) + parser.add_argument("--timeout-seconds", type=float, default=900.0) + parser.add_argument("--poll-seconds", type=float, default=2.0) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.timeout_seconds < 0 or args.poll_seconds <= 0: + raise SystemExit("timeouts must be non-negative and polling must be positive") + expected = configured_migration_heads( + args.database_url, + migration_track=args.migration_track, + ) + deadline = time.monotonic() + args.timeout_seconds + last_error = "" + while True: + try: + actual = database_migration_heads(args.database_url) + if actual == expected: + print("Database migration heads are ready: " + ",".join(actual)) + return 0 + last_error = ( + f"database heads={','.join(actual) or ''}; " + f"expected={','.join(expected) or ''}" + ) + except Exception as exc: # noqa: BLE001 - connection may become ready later + last_error = f"{type(exc).__name__}: {exc}" + if time.monotonic() >= deadline: + print( + "Database did not reach configured migration heads: " + last_error, + file=sys.stderr, + ) + return 75 + time.sleep(min(args.poll_seconds, max(0.05, deadline - time.monotonic()))) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/govoplan_core/core/access.py b/src/govoplan_core/core/access.py index 180110e..44ba31d 100644 --- a/src/govoplan_core/core/access.py +++ b/src/govoplan_core/core/access.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import Literal, Protocol, cast, runtime_checkable from govoplan_core.core.modules import AccessDecision +from govoplan_core.core.institutional import GovernedContextEnvelope ACCESS_MODULE_ID = "access" @@ -377,6 +378,7 @@ class AuditEvent: occurred_at: datetime | None = None correlation_id: str | None = None causation_id: str | None = None + institutional_context: GovernedContextEnvelope | None = None details: Mapping[str, object] = field(default_factory=dict) diff --git a/src/govoplan_core/core/automation.py b/src/govoplan_core/core/automation.py index 02de25b..e78e823 100644 --- a/src/govoplan_core/core/automation.py +++ b/src/govoplan_core/core/automation.py @@ -8,6 +8,7 @@ from typing import Literal, Protocol, runtime_checkable from govoplan_core.core.access import ( CAPABILITY_AUTH_AUTOMATION_PRINCIPAL_PROVIDER, ) +from govoplan_core.core.institutional import GovernedContextEnvelope AutomationInvocationKind = Literal[ "manual", @@ -110,6 +111,7 @@ class ActionExecutionRequest: input: Mapping[str, object] idempotency_key: str invocation: AutomationInvocation + institutional_context: GovernedContextEnvelope | None = None actor_ref: str | None = None preview_ref: str | None = None metadata: Mapping[str, object] = field(default_factory=dict) diff --git a/src/govoplan_core/core/configuration_packages.py b/src/govoplan_core/core/configuration_packages.py index 5995894..2abdc88 100644 --- a/src/govoplan_core/core/configuration_packages.py +++ b/src/govoplan_core/core/configuration_packages.py @@ -7,7 +7,8 @@ from datetime import UTC, datetime from pathlib import Path import json import os -from typing import Any, Literal, Protocol, runtime_checkable +import re +from typing import Any, Literal, Protocol, cast, runtime_checkable from govoplan_core.core.module_package_catalog import ( _canonical_catalog_bytes, @@ -21,6 +22,12 @@ from govoplan_core.core.module_package_catalog import ( _load_private_key, _parse_trusted_keys, ) +from govoplan_core.core.external_references import ( + IntegrationMaturity, + SOURCE_AUTHORITY_MODES, + SourceAuthorityMode, + integration_maturity_rank, +) from govoplan_core.security.http_fetch import fetch_http_text @@ -28,6 +35,166 @@ CONFIGURATION_PROVIDER_CAPABILITY = "configuration.provider" DiagnosticSeverity = Literal["blocker", "warning", "info"] PlanAction = Literal["create", "update", "bind", "skip", "blocked", "noop"] +ConfigurationPackageClass = Literal[ + "reference", + "product", + "sector", + "deployment", + "integration", +] +ConfigurationPackageEvidenceKind = Literal[ + "target_test", + "migration", + "upgrade", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "documentation", +] + +CONFIGURATION_PACKAGE_CLASSES: tuple[ConfigurationPackageClass, ...] = ( + "reference", + "product", + "sector", + "deployment", + "integration", +) +CONFIGURATION_PACKAGE_EVIDENCE_KINDS: tuple[ + ConfigurationPackageEvidenceKind, ... +] = ( + "target_test", + "migration", + "upgrade", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "documentation", +) +_SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") + + +@dataclass(frozen=True, slots=True) +class ConfigurationPackageParent: + package_id: str + version: str + relation: Literal["derived_from", "specializes", "extends"] = "derived_from" + + def __post_init__(self) -> None: + if not self.package_id.strip() or not self.version.strip(): + raise ValueError("Configuration package parent id and version are required.") + if self.relation not in {"derived_from", "specializes", "extends"}: + raise ValueError( + f"Unsupported configuration package parent relation: {self.relation!r}." + ) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageParent": + relation = _optional_str(value, "relation") or "derived_from" + if relation not in {"derived_from", "specializes", "extends"}: + raise ValueError(f"Unsupported configuration package parent relation: {relation!r}.") + return cls( + package_id=_required_str(value, "package_id"), + version=_required_str(value, "version"), + relation=relation, + ) + + def to_dict(self) -> dict[str, str]: + return { + "package_id": self.package_id, + "version": self.version, + "relation": self.relation, + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationPackageEvidence: + kind: ConfigurationPackageEvidenceKind + reference: str + summary: str + checksum: str | None = None + + def __post_init__(self) -> None: + if self.kind not in CONFIGURATION_PACKAGE_EVIDENCE_KINDS: + raise ValueError( + f"Unsupported configuration package evidence kind: {self.kind!r}." + ) + if not self.reference.strip() or not self.summary.strip(): + raise ValueError( + "Configuration package evidence reference and summary are required." + ) + if self.checksum is not None and not _SHA256_RE.fullmatch(self.checksum): + raise ValueError( + "Configuration package evidence checksum must use sha256:<64 lowercase hex>." + ) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageEvidence": + kind = _required_str(value, "kind") + if kind not in CONFIGURATION_PACKAGE_EVIDENCE_KINDS: + raise ValueError(f"Unsupported configuration package evidence kind: {kind!r}.") + return cls( + kind=kind, + reference=_required_str(value, "reference"), + summary=_required_str(value, "summary"), + checksum=_optional_str(value, "checksum"), + ) + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "reference": self.reference, + "summary": self.summary, + "checksum": self.checksum, + } + + +@dataclass(frozen=True, slots=True) +class ConfigurationProviderExpectation: + provider_id: str + authority_mode: SourceAuthorityMode + minimum_maturity: IntegrationMaturity + binding_ref: str | None = None + health_expectation: str = "healthy" + freshness_expectation: str | None = None + recovery_expectation: str | None = None + + def __post_init__(self) -> None: + if not self.provider_id.strip(): + raise ValueError("Configuration provider expectation id is required.") + if self.authority_mode not in SOURCE_AUTHORITY_MODES: + raise ValueError( + f"Unsupported provider authority mode: {self.authority_mode!r}." + ) + integration_maturity_rank(self.minimum_maturity) + if not self.health_expectation.strip(): + raise ValueError("Provider health expectation is required.") + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationProviderExpectation": + return cls( + provider_id=_required_str(value, "provider_id"), + authority_mode=_required_str(value, "authority_mode"), + minimum_maturity=_required_str(value, "minimum_maturity"), + binding_ref=_optional_str(value, "binding_ref"), + health_expectation=_optional_str(value, "health_expectation") or "healthy", + freshness_expectation=_optional_str(value, "freshness_expectation"), + recovery_expectation=_optional_str(value, "recovery_expectation"), + ) + + def to_dict(self) -> dict[str, object]: + return { + "provider_id": self.provider_id, + "authority_mode": self.authority_mode, + "minimum_maturity": self.minimum_maturity, + "binding_ref": self.binding_ref, + "health_expectation": self.health_expectation, + "freshness_expectation": self.freshness_expectation, + "recovery_expectation": self.recovery_expectation, + } @dataclass(frozen=True, slots=True) @@ -75,6 +242,7 @@ class ConfigurationPackageManifest: package_id: str name: str version: str + package_class: ConfigurationPackageClass = "product" description: str | None = None publisher: str | None = None category: str | None = None @@ -88,6 +256,29 @@ class ConfigurationPackageManifest: artifact_ref: str | None = None artifact_sha256: str | None = None signature: Mapping[str, Any] | None = None + parents: tuple[ConfigurationPackageParent, ...] = () + evidence: tuple[ConfigurationPackageEvidence, ...] = () + provider_expectations: tuple[ConfigurationProviderExpectation, ...] = () + + def __post_init__(self) -> None: + if self.package_class not in CONFIGURATION_PACKAGE_CLASSES: + raise ValueError( + f"Unsupported configuration package class: {self.package_class!r}." + ) + parent_keys = {(item.package_id, item.version) for item in self.parents} + if len(parent_keys) != len(self.parents): + raise ValueError("Configuration package parents must be unique.") + evidence_keys = {(item.kind, item.reference) for item in self.evidence} + if len(evidence_keys) != len(self.evidence): + raise ValueError("Configuration package evidence must be unique.") + provider_ids = [item.provider_id for item in self.provider_expectations] + if len(provider_ids) != len(set(provider_ids)): + raise ValueError("Configuration package provider expectations must be unique.") + for expectation in self.provider_expectations: + integration_maturity_rank(expectation.minimum_maturity) + issues = configuration_package_claim_issues(self) + if issues: + raise ValueError("Invalid configuration package claim: " + "; ".join(issues)) @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> "ConfigurationPackageManifest": @@ -95,6 +286,7 @@ class ConfigurationPackageManifest: package_id=_required_str(value, "package_id"), name=_required_str(value, "name"), version=_required_str(value, "version"), + package_class=_optional_str(value, "package_class") or "product", description=_optional_str(value, "description"), publisher=_optional_str(value, "publisher"), category=_optional_str(value, "category"), @@ -108,6 +300,21 @@ class ConfigurationPackageManifest: artifact_ref=_optional_str(value, "artifact_ref"), artifact_sha256=_optional_str(value, "artifact_sha256"), signature=value.get("signature") if isinstance(value.get("signature"), Mapping) else None, + parents=tuple( + ConfigurationPackageParent.from_mapping(item) + for item in _object_list(value.get("parents"), field_name="parents") + ), + evidence=tuple( + ConfigurationPackageEvidence.from_mapping(item) + for item in _object_list(value.get("evidence"), field_name="evidence") + ), + provider_expectations=tuple( + ConfigurationProviderExpectation.from_mapping(item) + for item in _object_list( + value.get("provider_expectations"), + field_name="provider_expectations", + ) + ), ) def to_dict(self) -> dict[str, object]: @@ -115,12 +322,18 @@ class ConfigurationPackageManifest: "package_id": self.package_id, "name": self.name, "version": self.version, + "package_class": self.package_class, "required_modules": [item.to_dict() for item in self.required_modules], "required_capabilities": list(self.required_capabilities), "optional_modules": [item.to_dict() for item in self.optional_modules], "fragments": [item.to_dict() for item in self.fragments], "data_requirements": [dict(item) for item in self.data_requirements], "tags": list(self.tags), + "parents": [item.to_dict() for item in self.parents], + "evidence": [item.to_dict() for item in self.evidence], + "provider_expectations": [ + item.to_dict() for item in self.provider_expectations + ], } for key, value in ( ("description", self.description), @@ -221,6 +434,12 @@ class ConfigurationPreflightContext: supplied_data: Mapping[str, Any] = field(default_factory=dict) installed_modules: Mapping[str, str] = field(default_factory=dict) capabilities: frozenset[str] = frozenset() + external_provider_declarations: Mapping[str, Mapping[str, Any]] = field( + default_factory=dict + ) + external_provider_states: Mapping[str, Mapping[str, Any]] = field( + default_factory=dict + ) dry_run: bool = True @@ -286,6 +505,7 @@ def dry_run_configuration_package( diagnostics.extend(_module_requirement_diagnostics(manifest, context)) diagnostics.extend(_capability_requirement_diagnostics(manifest, context)) + diagnostics.extend(_provider_expectation_diagnostics(manifest, context)) for item in manifest.data_requirements: requirement = ConfigurationRequiredData.from_mapping(item) required_data.append(requirement) @@ -369,6 +589,8 @@ def apply_configuration_package( supplied_data=supplied_data if supplied_data is not None else context.supplied_data, installed_modules=context.installed_modules, capabilities=context.capabilities, + external_provider_declarations=context.external_provider_declarations, + external_provider_states=context.external_provider_states, dry_run=False, ) preflight = dry_run_configuration_package(manifest, providers, apply_context) @@ -665,6 +887,154 @@ def record_configuration_package_catalog_acceptance(validation: dict[str, object state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") +def configuration_package_claim_issues( + manifest: ConfigurationPackageManifest, +) -> tuple[str, ...]: + evidence_kinds = {item.kind for item in manifest.evidence} + required_evidence: dict[str, frozenset[str]] = { + "reference": frozenset( + { + "target_test", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "documentation", + } + ), + "product": frozenset(), + "sector": frozenset({"documentation"}), + "deployment": frozenset( + {"target_test", "recovery", "security", "operations"} + ), + "integration": frozenset( + {"target_test", "recovery", "operations", "documentation"} + ), + } + issues: list[str] = [] + missing = sorted(required_evidence[manifest.package_class] - evidence_kinds) + if missing: + issues.append( + f"{manifest.package_class} package is missing evidence: " + + ", ".join(missing) + ) + if manifest.package_class in {"reference", "deployment", "integration"}: + unbound = sorted( + item.kind + for item in manifest.evidence + if item.kind != "documentation" and item.checksum is None + ) + if unbound: + issues.append( + f"{manifest.package_class} package has evidence without checksums: " + + ", ".join(unbound) + ) + if manifest.package_class == "sector" and not manifest.parents: + issues.append("sector packages must declare a parent package/version") + if manifest.package_class == "integration" and not manifest.provider_expectations: + issues.append("integration packages must declare external provider expectations") + return tuple(issues) + + +def validate_configuration_package_derivation( + child: ConfigurationPackageManifest, + parent: ConfigurationPackageManifest, +) -> tuple[ConfigurationDiagnostic, ...]: + diagnostics: list[ConfigurationDiagnostic] = [] + if not any( + item.package_id == parent.package_id and item.version == parent.version + for item in child.parents + ): + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="package_parent_provenance_missing", + message=( + f"Package {child.package_id!r} does not declare parent " + f"{parent.package_id}@{parent.version}." + ), + object_ref=parent.package_id, + ) + ) + child_modules = {item.module_id: item for item in child.required_modules} + for requirement in parent.required_modules: + candidate = child_modules.get(requirement.module_id) + if candidate is None or ( + requirement.version is not None + and candidate.version != requirement.version + ): + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="package_parent_module_constraint_loosened", + message=( + f"Derived package loosens parent module requirement " + f"{requirement.module_id!r}." + ), + module_id=requirement.module_id, + ) + ) + for capability in set(parent.required_capabilities) - set( + child.required_capabilities + ): + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="package_parent_capability_constraint_loosened", + message=( + f"Derived package removes required capability {capability!r}." + ), + object_ref=capability, + ) + ) + child_providers = { + item.provider_id: item for item in child.provider_expectations + } + for expectation in parent.provider_expectations: + candidate = child_providers.get(expectation.provider_id) + if candidate is None: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="package_parent_provider_constraint_removed", + message=( + f"Derived package removes provider expectation " + f"{expectation.provider_id!r}." + ), + object_ref=expectation.provider_id, + ) + ) + continue + if candidate.authority_mode != expectation.authority_mode: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="package_parent_authority_mode_changed", + message=( + f"Derived package changes authority mode for provider " + f"{expectation.provider_id!r}." + ), + object_ref=expectation.provider_id, + ) + ) + if integration_maturity_rank( + candidate.minimum_maturity + ) < integration_maturity_rank(expectation.minimum_maturity): + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="package_parent_provider_maturity_loosened", + message=( + f"Derived package lowers provider maturity for " + f"{expectation.provider_id!r}." + ), + object_ref=expectation.provider_id, + ) + ) + return tuple(diagnostics) + + def _configuration_package_manifest(package: ConfigurationPackageManifest | Mapping[str, Any]) -> ConfigurationPackageManifest: if isinstance(package, ConfigurationPackageManifest): return package @@ -716,6 +1086,194 @@ def _capability_requirement_diagnostics(manifest: ConfigurationPackageManifest, return diagnostics +def _provider_expectation_diagnostics( + manifest: ConfigurationPackageManifest, + context: ConfigurationPreflightContext, +) -> list[ConfigurationDiagnostic]: + diagnostics: list[ConfigurationDiagnostic] = [] + for expectation in manifest.provider_expectations: + declaration = context.external_provider_declarations.get( + expectation.provider_id + ) + if declaration is None: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_missing", + message=( + f"Required external provider {expectation.provider_id!r} " + "is not installed or declared." + ), + object_ref=expectation.provider_id, + resolution=( + "Install and enable a module exposing the declared provider." + ), + ) + ) + continue + supported_modes = { + str(item) + for item in declaration.get("authority_modes", ()) + if str(item).strip() + } + if expectation.authority_mode not in supported_modes: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_authority_mode_unsupported", + message=( + f"Provider {expectation.provider_id!r} does not support " + f"authority mode {expectation.authority_mode!r}." + ), + object_ref=expectation.provider_id, + ) + ) + actual_maturity = str(declaration.get("maturity") or "discover") + try: + maturity_sufficient = integration_maturity_rank( + cast(IntegrationMaturity, actual_maturity) + ) >= integration_maturity_rank(expectation.minimum_maturity) + except ValueError: + maturity_sufficient = False + if not maturity_sufficient: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_maturity_insufficient", + message=( + f"Provider {expectation.provider_id!r} has maturity " + f"{actual_maturity!r}; {expectation.minimum_maturity!r} is required." + ), + object_ref=expectation.provider_id, + ) + ) + provider_state = context.external_provider_states.get( + expectation.provider_id + ) + if provider_state is None: + diagnostics.append( + ConfigurationDiagnostic( + severity="warning", + code="external_provider_health_unverified", + message=( + f"Provider {expectation.provider_id!r} has no current " + "health/freshness observation." + ), + object_ref=expectation.provider_id, + ) + ) + continue + state = provider_state + if expectation.binding_ref is not None: + bindings = provider_state.get("bindings") + matching_binding = next( + ( + item + for item in bindings + if isinstance(item, Mapping) + and str(item.get("binding_ref") or "") + == expectation.binding_ref + ), + None, + ) if isinstance(bindings, Sequence) and not isinstance( + bindings, (str, bytes) + ) else None + if matching_binding is None and str( + provider_state.get("binding_ref") or "" + ) == expectation.binding_ref: + matching_binding = provider_state + if matching_binding is None: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_binding_mismatch", + message=( + f"Provider {expectation.provider_id!r} is not observed through " + f"required binding {expectation.binding_ref!r}." + ), + object_ref=expectation.provider_id, + ) + ) + continue + state = matching_binding + health = str(state.get("health") or state.get("health_state") or "unknown") + accepted_health = ( + {"ok", "healthy"} + if expectation.health_expectation == "healthy" + else {expectation.health_expectation} + ) + if health not in accepted_health: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_unhealthy", + message=( + f"Provider {expectation.provider_id!r} health is {health!r}." + ), + object_ref=expectation.provider_id, + resolution="Restore provider health or use a documented degraded path.", + ) + ) + observed_authority_mode = str(state.get("authority_mode") or "") + if ( + observed_authority_mode + and observed_authority_mode != expectation.authority_mode + ): + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_binding_authority_mismatch", + message=( + f"Provider {expectation.provider_id!r} is configured as " + f"{observed_authority_mode!r}; {expectation.authority_mode!r} " + "is required." + ), + object_ref=expectation.provider_id, + ) + ) + if expectation.freshness_expectation is not None: + freshness = str( + state.get("freshness") + or state.get("freshness_state") + or "unknown" + ) + if freshness != expectation.freshness_expectation: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_freshness_expectation_failed", + message=( + f"Provider {expectation.provider_id!r} freshness is " + f"{freshness!r}; {expectation.freshness_expectation!r} is required." + ), + object_ref=expectation.provider_id, + ) + ) + if expectation.recovery_expectation is not None: + behavior = declaration.get("behavior") + declared_recovery = ( + behavior.get(expectation.recovery_expectation) + if isinstance(behavior, Mapping) + else None + ) + observed_recovery = state.get("recovery") or state.get( + "recovery_state" + ) + if not declared_recovery and observed_recovery != expectation.recovery_expectation: + diagnostics.append( + ConfigurationDiagnostic( + severity="blocker", + code="external_provider_recovery_expectation_failed", + message=( + f"Provider {expectation.provider_id!r} does not satisfy " + f"recovery expectation {expectation.recovery_expectation!r}." + ), + object_ref=expectation.provider_id, + ) + ) + return diagnostics + + def _dedupe_diagnostics(items: Sequence[ConfigurationDiagnostic]) -> list[ConfigurationDiagnostic]: seen: set[tuple[object, ...]] = set() result: list[ConfigurationDiagnostic] = [] diff --git a/src/govoplan_core/core/datasources.py b/src/govoplan_core/core/datasources.py index f1f75ad..de76f72 100644 --- a/src/govoplan_core/core/datasources.py +++ b/src/govoplan_core/core/datasources.py @@ -5,6 +5,11 @@ from dataclasses import dataclass, field from datetime import datetime from typing import Literal, Protocol, runtime_checkable +from govoplan_core.core.external_references import ( + SOURCE_AUTHORITY_MODES, + SourceAuthorityMode, +) + CAPABILITY_DATASOURCE_CATALOGUE = "datasources.catalogue" CAPABILITY_DATASOURCE_LIFECYCLE = "datasources.lifecycle" @@ -53,6 +58,137 @@ class DatasourceField: nullable: bool = True +@dataclass(frozen=True, slots=True) +class DatasourceGovernance: + """Provider-neutral governance facts attached to a datasource revision.""" + + owner_ref: str | None = None + steward_ref: str | None = None + responsible_organization_ref: str | None = None + responsible_function_ref: str | None = None + authoritative_source_ref: str | None = None + authority_mode: SourceAuthorityMode = "linked_reference" + legal_basis_refs: tuple[str, ...] = () + purposes: tuple[str, ...] = () + semantic_definition: str | None = None + schema_owner_ref: str | None = None + official_keys: tuple[str, ...] = () + classification: str = "internal" + privacy_profile_ref: str | None = None + retention_policy_ref: str | None = None + hold_refs: tuple[str, ...] = () + publication_state: str = "draft" + transfer_agreement_ref: str | None = None + freshness_policy: Mapping[str, object] = field(default_factory=dict) + quality_policy: Mapping[str, object] = field(default_factory=dict) + known_limits: tuple[str, ...] = () + correction_procedure_ref: str | None = None + affected_refs: tuple[str, ...] = () + dependency_refs: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.authority_mode not in SOURCE_AUTHORITY_MODES: + raise DatasourceValidationError( + f"Unsupported datasource authority mode: {self.authority_mode!r}." + ) + if not self.classification.strip(): + raise DatasourceValidationError("Datasource classification is required.") + if not self.publication_state.strip(): + raise DatasourceValidationError("Datasource publication state is required.") + for field_name in ( + "legal_basis_refs", + "purposes", + "official_keys", + "hold_refs", + "known_limits", + "affected_refs", + "dependency_refs", + ): + values = getattr(self, field_name) + if any(not value.strip() for value in values): + raise DatasourceValidationError( + f"Datasource governance {field_name} cannot contain empty values." + ) + if len(values) != len(set(values)): + raise DatasourceValidationError( + f"Datasource governance {field_name} cannot contain duplicates." + ) + + @classmethod + def from_mapping(cls, value: Mapping[str, object] | None) -> "DatasourceGovernance": + source = value or {} + return cls( + owner_ref=_optional_governance_text(source.get("owner_ref")), + steward_ref=_optional_governance_text(source.get("steward_ref")), + responsible_organization_ref=_optional_governance_text( + source.get("responsible_organization_ref") + ), + responsible_function_ref=_optional_governance_text( + source.get("responsible_function_ref") + ), + authoritative_source_ref=_optional_governance_text( + source.get("authoritative_source_ref") + ), + authority_mode=str( + source.get("authority_mode") or "linked_reference" + ), # type: ignore[arg-type] + legal_basis_refs=_governance_texts(source.get("legal_basis_refs")), + purposes=_governance_texts(source.get("purposes")), + semantic_definition=_optional_governance_text( + source.get("semantic_definition") + ), + schema_owner_ref=_optional_governance_text(source.get("schema_owner_ref")), + official_keys=_governance_texts(source.get("official_keys")), + classification=str(source.get("classification") or "internal"), + privacy_profile_ref=_optional_governance_text( + source.get("privacy_profile_ref") + ), + retention_policy_ref=_optional_governance_text( + source.get("retention_policy_ref") + ), + hold_refs=_governance_texts(source.get("hold_refs")), + publication_state=str(source.get("publication_state") or "draft"), + transfer_agreement_ref=_optional_governance_text( + source.get("transfer_agreement_ref") + ), + freshness_policy=_governance_mapping(source.get("freshness_policy")), + quality_policy=_governance_mapping(source.get("quality_policy")), + known_limits=_governance_texts(source.get("known_limits")), + correction_procedure_ref=_optional_governance_text( + source.get("correction_procedure_ref") + ), + affected_refs=_governance_texts(source.get("affected_refs")), + dependency_refs=_governance_texts(source.get("dependency_refs")), + ) + + def to_dict(self) -> dict[str, object]: + return { + "owner_ref": self.owner_ref, + "steward_ref": self.steward_ref, + "responsible_organization_ref": self.responsible_organization_ref, + "responsible_function_ref": self.responsible_function_ref, + "authoritative_source_ref": self.authoritative_source_ref, + "authority_mode": self.authority_mode, + "legal_basis_refs": list(self.legal_basis_refs), + "purposes": list(self.purposes), + "semantic_definition": self.semantic_definition, + "schema_owner_ref": self.schema_owner_ref, + "official_keys": list(self.official_keys), + "classification": self.classification, + "privacy_profile_ref": self.privacy_profile_ref, + "retention_policy_ref": self.retention_policy_ref, + "hold_refs": list(self.hold_refs), + "publication_state": self.publication_state, + "transfer_agreement_ref": self.transfer_agreement_ref, + "freshness_policy": dict(self.freshness_policy), + "quality_policy": dict(self.quality_policy), + "known_limits": list(self.known_limits), + "correction_procedure_ref": self.correction_procedure_ref, + "affected_refs": list(self.affected_refs), + "dependency_refs": list(self.dependency_refs), + } + + @dataclass(frozen=True, slots=True) class DatasourceDescriptor: ref: str @@ -75,6 +211,7 @@ class DatasourceDescriptor: capabilities: tuple[str, ...] = ("read",) provenance: Mapping[str, object] = field(default_factory=dict) metadata: Mapping[str, object] = field(default_factory=dict) + governance: DatasourceGovernance = field(default_factory=DatasourceGovernance) @dataclass(frozen=True, slots=True) @@ -93,6 +230,7 @@ class DatasourceMaterialization: created_at: datetime | None = None provenance: Mapping[str, object] = field(default_factory=dict) metadata: Mapping[str, object] = field(default_factory=dict) + governance: DatasourceGovernance = field(default_factory=DatasourceGovernance) @dataclass(frozen=True, slots=True) @@ -115,6 +253,7 @@ class DatasourceStage: promoted_materialization_ref: str | None = None provenance: Mapping[str, object] = field(default_factory=dict) metadata: Mapping[str, object] = field(default_factory=dict) + governance: DatasourceGovernance = field(default_factory=DatasourceGovernance) @dataclass(frozen=True, slots=True) @@ -151,6 +290,7 @@ class DatasourceStageInput: provider_ref: str | None = None provenance: Mapping[str, object] = field(default_factory=dict) metadata: Mapping[str, object] = field(default_factory=dict) + governance: DatasourceGovernance | None = None @dataclass(frozen=True, slots=True) @@ -169,6 +309,7 @@ class DatasourcePublicationRequest: source_timestamp: datetime | None = None provenance: Mapping[str, object] = field(default_factory=dict) metadata: Mapping[str, object] = field(default_factory=dict) + governance: DatasourceGovernance | None = None @dataclass(frozen=True, slots=True) @@ -226,6 +367,13 @@ class DatasourceCatalogueProvider(Protocol): *, query: str = "", limit: int = 100, + authority_mode: str | None = None, + classification: str | None = None, + publication_state: str | None = None, + owner_ref: str | None = None, + responsible_organization_ref: str | None = None, + affected_ref: str | None = None, + dependency_ref: str | None = None, ) -> Sequence[DatasourceDescriptor]: ... @@ -298,6 +446,17 @@ class DatasourceLifecycleProvider(Protocol): source_name: str, mode: DatasourceMode, description: str | None = None, + governance: DatasourceGovernance | None = None, + ) -> DatasourceDescriptor: + ... + + def update_datasource_governance( + self, + session: object, + principal: object, + *, + datasource_ref: str, + governance: DatasourceGovernance, ) -> DatasourceDescriptor: ... @@ -406,6 +565,25 @@ def _capability(registry: object | None, name: str) -> object | None: return registry.capability(name) +def _optional_governance_text(value: object) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None + + +def _governance_texts(value: object) -> tuple[str, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return () + return tuple(str(item).strip() for item in value) + + +def _governance_mapping(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return {} + return {str(key): item for key, item in value.items()} + + __all__ = [ "CAPABILITY_DATASOURCE_CATALOGUE", "CAPABILITY_DATASOURCE_LIFECYCLE", @@ -416,6 +594,7 @@ __all__ = [ "DatasourceDescriptor", "DatasourceError", "DatasourceField", + "DatasourceGovernance", "DatasourceKind", "DatasourceLifecycleProvider", "DatasourceMaterialization", diff --git a/src/govoplan_core/core/events.py b/src/govoplan_core/core/events.py index a3d9a19..a576848 100644 --- a/src/govoplan_core/core/events.py +++ b/src/govoplan_core/core/events.py @@ -13,6 +13,8 @@ import uuid from sqlalchemy import event as sqlalchemy_event from sqlalchemy.orm import Session +from govoplan_core.core.institutional import GovernedContextEnvelope + _TRACE_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") _CONSUMER_ID_RE = re.compile(r"^[a-z][a-z0-9_.:-]{0,127}$") @@ -87,6 +89,7 @@ class PlatformEvent: subject: EventObjectRef | None = None resource: EventObjectRef | None = None classification: EventClassification = "internal" + institutional_context: GovernedContextEnvelope | None = None def to_dict(self) -> dict[str, Any]: return { @@ -102,6 +105,11 @@ class PlatformEvent: "subject": self.subject.to_dict() if self.subject else None, "resource": self.resource.to_dict() if self.resource else None, "classification": self.classification, + "institutional_context": ( + self.institutional_context.to_dict() + if self.institutional_context is not None + else None + ), } @@ -254,6 +262,7 @@ def ensure_event_trace(event: PlatformEvent) -> PlatformEvent: subject=event.subject, resource=event.resource, classification=event.classification, + institutional_context=event.institutional_context, ) diff --git a/src/govoplan_core/core/external_references.py b/src/govoplan_core/core/external_references.py index 2ed99a4..784a46e 100644 --- a/src/govoplan_core/core/external_references.py +++ b/src/govoplan_core/core/external_references.py @@ -17,6 +17,14 @@ IntegrationMaturity = Literal[ "migrate", "replace", ] +SourceAuthorityMode = Literal[ + "native_authoritative", + "external_authoritative", + "external_mirror", + "governed_sync", + "governance_overlay", + "linked_reference", +] INTEGRATION_MATURITY_ORDER: tuple[IntegrationMaturity, ...] = ( "discover", @@ -28,6 +36,14 @@ INTEGRATION_MATURITY_ORDER: tuple[IntegrationMaturity, ...] = ( "migrate", "replace", ) +SOURCE_AUTHORITY_MODES: tuple[SourceAuthorityMode, ...] = ( + "native_authoritative", + "external_authoritative", + "external_mirror", + "governed_sync", + "governance_overlay", + "linked_reference", +) class ExternalReferenceValidationError(ValueError): @@ -42,6 +58,7 @@ class ExternalObjectReference: object_type: str object_id: str maturity: IntegrationMaturity = "link" + authority_mode: SourceAuthorityMode = "linked_reference" connector_id: str | None = None canonical_url: str | None = None version: str | None = None @@ -65,6 +82,24 @@ class ExternalObjectReference: raise ExternalReferenceValidationError( f"Unsupported integration maturity: {self.maturity!r}." ) + if self.authority_mode not in SOURCE_AUTHORITY_MODES: + raise ExternalReferenceValidationError( + f"Unsupported source-authority mode: {self.authority_mode!r}." + ) + if ( + self.authority_mode == "external_mirror" + and not self.supports("read") + ): + raise ExternalReferenceValidationError( + "External-mirror references require read maturity or higher." + ) + if ( + self.authority_mode == "governed_sync" + and not self.supports("synchronize") + ): + raise ExternalReferenceValidationError( + "Governed-sync references require synchronize maturity or higher." + ) if self.connector_id is not None: connector_id = self.connector_id.strip() if not connector_id: @@ -94,6 +129,7 @@ class ExternalObjectReference: "object_type": self.object_type, "object_id": self.object_id, "maturity": self.maturity, + "authority_mode": self.authority_mode, "connector_id": self.connector_id, "canonical_url": self.canonical_url, "version": self.version, @@ -133,5 +169,7 @@ __all__ = [ "ExternalReferenceValidationError", "INTEGRATION_MATURITY_ORDER", "IntegrationMaturity", + "SOURCE_AUTHORITY_MODES", + "SourceAuthorityMode", "integration_maturity_rank", ] diff --git a/src/govoplan_core/core/install_config.py b/src/govoplan_core/core/install_config.py index 90ab051..e8d8dd7 100644 --- a/src/govoplan_core/core/install_config.py +++ b/src/govoplan_core/core/install_config.py @@ -90,7 +90,9 @@ class _ConfigIssueCollector: def add(self, level: ConfigIssueLevel, key: str, message: str, action: str) -> None: if self.strict and level == "warning": level = "error" - self.issues.append(ConfigIssue(level=level, key=key, message=message, action=action)) + self.issues.append( + ConfigIssue(level=level, key=key, message=message, action=action) + ) _LOCAL_PROFILES = {"dev", "local", "local-dev", "test"} @@ -120,9 +122,15 @@ def generate_master_key() -> str: return Fernet.generate_key().decode("ascii") -def env_template(*, profile: str = "self-hosted", generate_secrets: bool = False) -> str: +def env_template( + *, profile: str = "self-hosted", generate_secrets: bool = False +) -> str: clean_profile = normalize_install_profile(profile) - master_key = generate_master_key() if generate_secrets else "" + master_key = ( + generate_master_key() + if generate_secrets + else "" + ) if clean_profile == "production-like": return _production_like_env_template(master_key) return _self_hosted_env_template(master_key) @@ -144,14 +152,19 @@ def validate_runtime_configuration( _validate_async_and_auth_settings(env, runtime, collector) _validate_cors_settings(env, runtime, collector) _validate_file_storage_settings(env, runtime, collector) + _validate_shared_state_settings(env, collector) _validate_outbound_connector_policy(env, runtime, collector) _validate_module_catalog_trust(env, runtime, collector) return ConfigValidationResult(profile=runtime.name, issues=tuple(collector.issues)) def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _RuntimeProfile: - clean_profile = normalize_install_profile(profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV")) - production = clean_profile in _PRODUCTION_PROFILES or env.get("APP_ENV", "").strip().lower() in {"prod", "production"} + clean_profile = normalize_install_profile( + profile or env.get("GOVOPLAN_INSTALL_PROFILE") or env.get("APP_ENV") + ) + production = clean_profile in _PRODUCTION_PROFILES or env.get( + "APP_ENV", "" + ).strip().lower() in {"prod", "production"} production_like = production or clean_profile in _PRODUCTION_LIKE_PROFILES return _RuntimeProfile( name=clean_profile, @@ -161,86 +174,222 @@ def _runtime_profile(env: Mapping[str, str], *, profile: str | None) -> _Runtime ) -def _validate_app_env(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_app_env( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: app_env = _clean(env.get("APP_ENV")) if not app_env and runtime.production_like: - collector.add("error", "APP_ENV", "APP_ENV is missing for a production-like install.", "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.") + collector.add( + "error", + "APP_ENV", + "APP_ENV is missing for a production-like install.", + "Set APP_ENV=staging, APP_ENV=production, or another explicit deployment profile.", + ) elif app_env.lower() in {"dev", "test", "local"} and runtime.production_like: - collector.add("error", "APP_ENV", f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.", "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.") + collector.add( + "error", + "APP_ENV", + f"APP_ENV={app_env!r} is not valid for profile {runtime.name!r}.", + "Use APP_ENV=staging for production-like testing or APP_ENV=production for a real deployment.", + ) -def _validate_database_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_database_settings( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: database_url = _clean(env.get("DATABASE_URL")) if not database_url: - collector.add("error", "DATABASE_URL", "DATABASE_URL is missing.", "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.") + collector.add( + "error", + "DATABASE_URL", + "DATABASE_URL is missing.", + "Set DATABASE_URL to the PostgreSQL SQLAlchemy URL used by the API and modules.", + ) return backend = _database_backend(database_url) if backend is None: - collector.add("error", "DATABASE_URL", "DATABASE_URL is not a valid SQLAlchemy URL.", "Use a value like postgresql+psycopg://user:password@host:5432/database.") + collector.add( + "error", + "DATABASE_URL", + "DATABASE_URL is not a valid SQLAlchemy URL.", + "Use a value like postgresql+psycopg://user:password@host:5432/database.", + ) return if backend == "sqlite" and runtime.production_like: - collector.add("error", "DATABASE_URL", "SQLite is only supported for disposable local development.", "Use PostgreSQL for production-like and self-hosted installs.") + collector.add( + "error", + "DATABASE_URL", + "SQLite is only supported for disposable local development.", + "Use PostgreSQL for production-like and self-hosted installs.", + ) elif backend != "postgresql" and runtime.production: - collector.add("warning", "DATABASE_URL", f"Database backend {backend!r} is not the preferred production target.", "Use PostgreSQL unless this deployment has an explicit support decision.") + collector.add( + "warning", + "DATABASE_URL", + f"Database backend {backend!r} is not the preferred production target.", + "Use PostgreSQL unless this deployment has an explicit support decision.", + ) if backend == "postgresql" and not _clean(env.get("GOVOPLAN_DATABASE_URL_PGTOOLS")): - collector.add("warning", "GOVOPLAN_DATABASE_URL_PGTOOLS", "PostgreSQL backup/restore tools URL is missing.", "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.") + collector.add( + "warning", + "GOVOPLAN_DATABASE_URL_PGTOOLS", + "PostgreSQL backup/restore tools URL is missing.", + "Set GOVOPLAN_DATABASE_URL_PGTOOLS to the same database without the SQLAlchemy driver marker, for example postgresql://user:password@host:5432/database.", + ) -def _validate_master_key(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_master_key( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: master_key = _clean(env.get("MASTER_KEY_B64")) if not master_key and not runtime.local: - collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 is required outside local dev/test.", "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.") + collector.add( + "error", + "MASTER_KEY_B64", + "MASTER_KEY_B64 is required outside local dev/test.", + "Generate a Fernet key with `govoplan-config env-template --generate-secrets` or `python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` and store it in deployment secrets.", + ) return if not master_key: return error = _master_key_error(master_key) if error: - collector.add("error", "MASTER_KEY_B64", error, "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.") + collector.add( + "error", + "MASTER_KEY_B64", + error, + "Replace MASTER_KEY_B64 with a Fernet key or base64-encoded 32-byte key.", + ) elif "change-me" in master_key.lower() or "generate" in master_key.lower(): - collector.add("error", "MASTER_KEY_B64", "MASTER_KEY_B64 still looks like a placeholder.", "Generate a real deployment key and store it outside git.") + collector.add( + "error", + "MASTER_KEY_B64", + "MASTER_KEY_B64 still looks like a placeholder.", + "Generate a real deployment key and store it outside git.", + ) -def _validate_enabled_modules(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_enabled_modules( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: enabled_modules = _csv(env.get("ENABLED_MODULES")) if not enabled_modules and runtime.production_like: - collector.add("error", "ENABLED_MODULES", "ENABLED_MODULES is missing.", "Set ENABLED_MODULES explicitly so startup module composition is intentional.") + collector.add( + "error", + "ENABLED_MODULES", + "ENABLED_MODULES is missing.", + "Set ENABLED_MODULES explicitly so startup module composition is intentional.", + ) elif "access" not in enabled_modules and runtime.production_like: - collector.add("error", "ENABLED_MODULES", "The access module is not enabled.", "Include `access` unless this deployment has a replacement auth/principal provider.") + collector.add( + "error", + "ENABLED_MODULES", + "The access module is not enabled.", + "Include `access` unless this deployment has a replacement auth/principal provider.", + ) elif enabled_modules and "admin" not in enabled_modules and runtime.production_like: - collector.add("warning", "ENABLED_MODULES", "The admin module is not enabled.", "Keep `admin` enabled for operator UI unless this is a deliberately headless install.") + collector.add( + "warning", + "ENABLED_MODULES", + "The admin module is not enabled.", + "Keep `admin` enabled for operator UI unless this is a deliberately headless install.", + ) -def _validate_async_and_auth_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_async_and_auth_settings( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: if _truthy(env.get("CELERY_ENABLED")) and not _clean(env.get("REDIS_URL")): - collector.add("error", "REDIS_URL", "CELERY_ENABLED=true but REDIS_URL is missing.", "Set REDIS_URL to the Redis broker/result backend used by workers.") + collector.add( + "error", + "REDIS_URL", + "CELERY_ENABLED=true but REDIS_URL is missing.", + "Set REDIS_URL to the Redis broker/result backend used by workers.", + ) if runtime.production and _truthy(env.get("DEV_BOOTSTRAP_ENABLED")): - collector.add("error", "DEV_BOOTSTRAP_ENABLED", "Development bootstrap is enabled in production.", "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.") + collector.add( + "error", + "DEV_BOOTSTRAP_ENABLED", + "Development bootstrap is enabled in production.", + "Set DEV_BOOTSTRAP_ENABLED=false and create first administrators through the controlled bootstrap path.", + ) if runtime.production and not _truthy(env.get("AUTH_COOKIE_SECURE")): - collector.add("error", "AUTH_COOKIE_SECURE", "Secure auth cookies are disabled for production.", "Set AUTH_COOKIE_SECURE=true behind HTTPS.") + collector.add( + "error", + "AUTH_COOKIE_SECURE", + "Secure auth cookies are disabled for production.", + "Set AUTH_COOKIE_SECURE=true behind HTTPS.", + ) -def _validate_cors_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_cors_settings( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: cors_origins = _csv(env.get("CORS_ORIGINS")) if runtime.production_like and not cors_origins: - collector.add("error", "CORS_ORIGINS", "CORS_ORIGINS is missing.", "Set CORS_ORIGINS to the exact WebUI origin or origins.") + collector.add( + "error", + "CORS_ORIGINS", + "CORS_ORIGINS is missing.", + "Set CORS_ORIGINS to the exact WebUI origin or origins.", + ) elif "*" in cors_origins and runtime.production_like: - collector.add("error", "CORS_ORIGINS", "Wildcard CORS is not allowed for production-like installs.", "Replace `*` with exact HTTPS/WebUI origins.") + collector.add( + "error", + "CORS_ORIGINS", + "Wildcard CORS is not allowed for production-like installs.", + "Replace `*` with exact HTTPS/WebUI origins.", + ) elif runtime.production and set(cors_origins) <= _DEFAULT_LOCAL_CORS: - collector.add("warning", "CORS_ORIGINS", "CORS_ORIGINS still contains only local development origins.", "Set CORS_ORIGINS to the deployed WebUI origin.") + collector.add( + "warning", + "CORS_ORIGINS", + "CORS_ORIGINS still contains only local development origins.", + "Set CORS_ORIGINS to the deployed WebUI origin.", + ) trusted_hosts = _csv(env.get("GOVOPLAN_TRUSTED_HOSTS")) if runtime.production_like and not trusted_hosts: - collector.add("error", "GOVOPLAN_TRUSTED_HOSTS", "Trusted HTTP hosts are not configured.", "Set GOVOPLAN_TRUSTED_HOSTS to the exact API host names accepted by this deployment.") + collector.add( + "error", + "GOVOPLAN_TRUSTED_HOSTS", + "Trusted HTTP hosts are not configured.", + "Set GOVOPLAN_TRUSTED_HOSTS to the exact API host names accepted by this deployment.", + ) elif "*" in trusted_hosts and runtime.production_like: - collector.add("error", "GOVOPLAN_TRUSTED_HOSTS", "Wildcard trusted hosts are not allowed for production-like installs.", "Replace `*` with exact host names or narrowly scoped `*.example.org` entries.") + collector.add( + "error", + "GOVOPLAN_TRUSTED_HOSTS", + "Wildcard trusted hosts are not allowed for production-like installs.", + "Replace `*` with exact host names or narrowly scoped `*.example.org` entries.", + ) forwarded_allow_ips = _csv(env.get("FORWARDED_ALLOW_IPS")) if runtime.production_like and "*" in forwarded_allow_ips: - collector.add("error", "FORWARDED_ALLOW_IPS", "Proxy headers must not be trusted from every address.", "Set FORWARDED_ALLOW_IPS to the reverse proxy address or network passed to Uvicorn.") + collector.add( + "error", + "FORWARDED_ALLOW_IPS", + "Proxy headers must not be trusted from every address.", + "Set FORWARDED_ALLOW_IPS to the reverse proxy address or network passed to Uvicorn.", + ) -def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_file_storage_settings( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: storage_backend = _clean(env.get("FILE_STORAGE_BACKEND")) or "local" - deployment_managed_raw = _clean(env.get("FILE_STORAGE_S3_DEPLOYMENT_MANAGED")).lower() - if deployment_managed_raw and deployment_managed_raw not in {"true", "false", "1", "0", "yes", "no", "on", "off"}: + deployment_managed_raw = _clean( + env.get("FILE_STORAGE_S3_DEPLOYMENT_MANAGED") + ).lower() + endpoint_trusted_raw = _clean(env.get("FILE_STORAGE_S3_ENDPOINT_TRUSTED")).lower() + if deployment_managed_raw and deployment_managed_raw not in { + "true", + "false", + "1", + "0", + "yes", + "no", + "on", + "off", + }: collector.add( "error", "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", @@ -248,13 +397,30 @@ def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimePro "Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false, or let the supported installer manage Garage.", ) deployment_managed = _truthy(deployment_managed_raw) + if endpoint_trusted_raw and endpoint_trusted_raw not in { + "true", + "false", + "1", + "0", + "yes", + "no", + "on", + "off", + }: + collector.add( + "error", + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + "External S3 endpoint trust must be an explicit boolean.", + "Set it only for a deployment-controlled HTTPS storage origin.", + ) + endpoint_trusted = _truthy(endpoint_trusted_raw) if storage_backend == "local": - if deployment_managed: + if deployment_managed or endpoint_trusted: collector.add( "error", - "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", - "Managed S3 trust cannot be enabled for local file storage.", - "Set FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false.", + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + "S3 endpoint trust cannot be enabled for local file storage.", + "Disable both S3 trust settings while FILE_STORAGE_BACKEND=local.", ) _validate_local_file_storage(env, runtime, collector) elif storage_backend == "s3": @@ -262,16 +428,34 @@ def _validate_file_storage_settings(env: Mapping[str, str], runtime: _RuntimePro env, collector, deployment_managed=deployment_managed, + endpoint_trusted=endpoint_trusted, ) else: - collector.add("error", "FILE_STORAGE_BACKEND", f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", "Use `local` or `s3`.") + collector.add( + "error", + "FILE_STORAGE_BACKEND", + f"Unsupported FILE_STORAGE_BACKEND={storage_backend!r}.", + "Use `local` or `s3`.", + ) -def _validate_local_file_storage(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: +def _validate_local_file_storage( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: if not _clean(env.get("FILE_STORAGE_LOCAL_ROOT")) and runtime.production_like: - collector.add("error", "FILE_STORAGE_LOCAL_ROOT", "Local file storage root is missing.", "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.") + collector.add( + "error", + "FILE_STORAGE_LOCAL_ROOT", + "Local file storage root is missing.", + "Set FILE_STORAGE_LOCAL_ROOT to a durable, backed-up path.", + ) elif runtime.production: - collector.add("warning", "FILE_STORAGE_BACKEND", "Production is configured for local file storage.", "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.") + collector.add( + "warning", + "FILE_STORAGE_BACKEND", + "Production is configured for local file storage.", + "Confirm the path is durable and backed up, or use object storage once the deployment needs independent file scaling.", + ) def _validate_s3_file_storage( @@ -279,10 +463,22 @@ def _validate_s3_file_storage( collector: _ConfigIssueCollector, *, deployment_managed: bool, + endpoint_trusted: bool, ) -> None: - for key in ("FILE_STORAGE_S3_ENDPOINT_URL", "FILE_STORAGE_S3_REGION", "FILE_STORAGE_S3_ACCESS_KEY_ID", "FILE_STORAGE_S3_SECRET_ACCESS_KEY", "FILE_STORAGE_S3_BUCKET"): + for key in ( + "FILE_STORAGE_S3_ENDPOINT_URL", + "FILE_STORAGE_S3_REGION", + "FILE_STORAGE_S3_ACCESS_KEY_ID", + "FILE_STORAGE_S3_SECRET_ACCESS_KEY", + "FILE_STORAGE_S3_BUCKET", + ): if not _clean(env.get(key)): - collector.add("error", key, f"{key} is required when FILE_STORAGE_BACKEND=s3.", "Configure all FILE_STORAGE_S3_* settings through deployment secrets.") + collector.add( + "error", + key, + f"{key} is required when FILE_STORAGE_BACKEND=s3.", + "Configure all FILE_STORAGE_S3_* settings through deployment secrets.", + ) if ( deployment_managed and _clean(env.get("FILE_STORAGE_S3_ENDPOINT_URL")) != "http://garage:3900" @@ -293,6 +489,126 @@ def _validate_s3_file_storage( "Installer-managed S3 trust is restricted to http://garage:3900.", "Use the exact managed Garage endpoint or disable FILE_STORAGE_S3_DEPLOYMENT_MANAGED.", ) + if deployment_managed and endpoint_trusted: + collector.add( + "error", + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + "Managed Garage trust and external endpoint trust are mutually exclusive.", + "Use installer-managed Garage trust or one explicit external endpoint.", + ) + if not deployment_managed and not endpoint_trusted: + collector.add( + "error", + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + "External S3 storage requires an explicit deployment trust decision.", + "Set FILE_STORAGE_S3_ENDPOINT_TRUSTED=true only for a deployment-controlled HTTPS origin.", + ) + endpoint = _clean(env.get("FILE_STORAGE_S3_ENDPOINT_URL")) + if endpoint_trusted and not endpoint.lower().startswith("https://"): + collector.add( + "error", + "FILE_STORAGE_S3_ENDPOINT_URL", + "Deployment-trusted external S3 storage must use HTTPS.", + "Use an HTTPS storage origin with certificate verification.", + ) + + +def _validate_shared_state_settings( + env: Mapping[str, str], + collector: _ConfigIssueCollector, +) -> None: + state_profile = (_clean(env.get("GOVOPLAN_STATE_PROFILE")) or "local").lower() + if state_profile not in {"local", "host-shared", "shared"}: + collector.add( + "error", + "GOVOPLAN_STATE_PROFILE", + f"Unsupported state profile {state_profile!r}.", + "Use `local` for one process per role, `host-shared` for one Compose host, or `shared` for a multi-host stateless tier.", + ) + return + try: + api_replicas = int(_clean(env.get("GOVOPLAN_EXPECTED_API_REPLICAS")) or "1") + worker_replicas = int( + _clean(env.get("GOVOPLAN_EXPECTED_WORKER_REPLICAS")) or "0" + ) + except ValueError: + collector.add( + "error", + "GOVOPLAN_EXPECTED_API_REPLICAS", + "Expected replica counts must be integers.", + "Set GOVOPLAN_EXPECTED_API_REPLICAS and GOVOPLAN_EXPECTED_WORKER_REPLICAS to non-negative integers.", + ) + return + if api_replicas < 1 or worker_replicas < 0: + collector.add( + "error", + "GOVOPLAN_EXPECTED_API_REPLICAS", + "Expected replica counts are outside their supported range.", + "Configure at least one API replica and zero or more worker replicas.", + ) + if state_profile == "local": + if api_replicas > 1 or worker_replicas > 1: + collector.add( + "error", + "GOVOPLAN_STATE_PROFILE", + "A local-state profile cannot safely run replicated API or worker nodes.", + "Use `host-shared` with one shared host volume, or `shared` with PostgreSQL, Redis, and S3-compatible object storage.", + ) + return + installation_id = _clean(env.get("GOVOPLAN_INSTALLATION_ID")) + if not installation_id or ( + state_profile == "shared" and installation_id == "govoplan-local" + ): + collector.add( + "error", + "GOVOPLAN_INSTALLATION_ID", + "Shared-state deployments require a stable installation identifier.", + "Set one immutable deployment-wide GOVOPLAN_INSTALLATION_ID on every node.", + ) + if _database_backend(_clean(env.get("DATABASE_URL"))) != "postgresql": + collector.add( + "error", + "DATABASE_URL", + "Shared-state deployments require PostgreSQL.", + "Point every API, scheduler, and worker node at the same logical PostgreSQL service.", + ) + if not _clean(env.get("REDIS_URL")): + collector.add( + "error", + "REDIS_URL", + "Shared-state deployments require a common Redis service.", + "Configure the same Redis endpoint for all API and worker nodes.", + ) + if ( + state_profile == "shared" + and (_clean(env.get("FILE_STORAGE_BACKEND")) or "local").lower() != "s3" + ): + collector.add( + "error", + "FILE_STORAGE_BACKEND", + "Shared-state deployments cannot use node-local object storage.", + "Set FILE_STORAGE_BACKEND=s3 and configure one shared S3-compatible bucket.", + ) + try: + heartbeat = int(_clean(env.get("GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS")) or "15") + stale_after = int( + _clean(env.get("GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS")) or "60" + ) + except ValueError: + collector.add( + "error", + "GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS", + "Runtime heartbeat and stale intervals must be integers.", + "Use a heartbeat interval shorter than one third of the stale interval.", + ) + else: + if heartbeat < 2 or stale_after < max(10, heartbeat * 3): + collector.add( + "error", + "GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS", + "Runtime stale detection leaves insufficient room for missed heartbeats.", + "Set stale-after to at least three heartbeat intervals and at least ten seconds.", + ) def _validate_outbound_connector_policy( @@ -300,8 +616,19 @@ def _validate_outbound_connector_policy( runtime: _RuntimeProfile, collector: _ConfigIssueCollector, ) -> None: - private_networks = _clean(env.get("GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS")).lower() - if runtime.production_like and private_networks not in {"true", "false", "1", "0", "yes", "no", "on", "off"}: + private_networks = _clean( + env.get("GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS") + ).lower() + if runtime.production_like and private_networks not in { + "true", + "false", + "1", + "0", + "yes", + "no", + "on", + "off", + }: collector.add( "error", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", @@ -321,13 +648,21 @@ def _validate_outbound_connector_policy( except ValueError: parsed = 0 if parsed <= 0: - collector.add("error", key, f"{key} must be a positive byte count.", "Use a positive integer byte limit.") + collector.add( + "error", + key, + f"{key} must be a positive byte count.", + "Use a positive integer byte limit.", + ) secret_env_names = [ item.strip() for item in env.get("GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST", "").split(",") if item.strip() ] - if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", item) is None for item in secret_env_names): + if any( + re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", item) is None + for item in secret_env_names + ): collector.add( "error", "GOVOPLAN_CONNECTOR_SECRET_ENV_ALLOWLIST", @@ -348,14 +683,28 @@ def _validate_outbound_connector_policy( ) -def _validate_module_catalog_trust(env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector) -> None: - catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG")) +def _validate_module_catalog_trust( + env: Mapping[str, str], runtime: _RuntimeProfile, collector: _ConfigIssueCollector +) -> None: + catalog_source = _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_URL")) or _clean( + env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG") + ) if not runtime.production or not catalog_source: return if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE")): - collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", "A module catalog source is configured without a trusted keyring file.", "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.") + collector.add( + "error", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE", + "A module catalog source is configured without a trusted keyring file.", + "Pin the published GovOPlaN catalog keyring locally and set GOVOPLAN_MODULE_PACKAGE_CATALOG_TRUSTED_KEYS_FILE.", + ) if not _clean(env.get("GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL")): - collector.add("error", "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", "A module catalog source is configured without an approved release channel.", "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.") + collector.add( + "error", + "GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL", + "A module catalog source is configured without an approved release channel.", + "Set GOVOPLAN_MODULE_PACKAGE_CATALOG_APPROVED_CHANNEL=stable or another approved deployment channel.", + ) def _self_hosted_env_template(master_key: str) -> str: @@ -364,6 +713,13 @@ def _self_hosted_env_template(master_key: str) -> str: APP_ENV=production GOVOPLAN_INSTALL_PROFILE=self-hosted +GOVOPLAN_INSTALLATION_ID=govoplan-production +GOVOPLAN_STATE_PROFILE=local +GOVOPLAN_RUNTIME_ROLE=api +GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS=15 +GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS=60 +GOVOPLAN_EXPECTED_API_REPLICAS=1 +GOVOPLAN_EXPECTED_WORKER_REPLICAS=1 MASTER_KEY_B64={master_key} DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan @@ -409,6 +765,7 @@ FILE_STORAGE_BACKEND=local FILE_STORAGE_LOCAL_ROOT=/var/lib/govoplan/files FILE_STORAGE_LOCAL_FALLBACK_ROOTS= FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false +FILE_STORAGE_S3_ENDPOINT_TRUSTED=false FILE_ARCHIVE_MAX_ENTRIES=10000 FILE_ARCHIVE_MAX_EXPANDED_BYTES=2147483648 FILE_ARCHIVE_MAX_EXPANSION_RATIO=100 @@ -430,6 +787,13 @@ def _production_like_env_template(master_key: str) -> str: APP_ENV=staging GOVOPLAN_INSTALL_PROFILE=production-like +GOVOPLAN_INSTALLATION_ID=govoplan-production-like +GOVOPLAN_STATE_PROFILE=local +GOVOPLAN_RUNTIME_ROLE=api +GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS=15 +GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS=60 +GOVOPLAN_EXPECTED_API_REPLICAS=1 +GOVOPLAN_EXPECTED_WORKER_REPLICAS=1 MASTER_KEY_B64={master_key} GOVOPLAN_PRODUCTION_LIKE_POSTGRES_DB=govoplan @@ -474,6 +838,7 @@ AUTH_COOKIE_SECURE=false FILE_STORAGE_BACKEND=local FILE_STORAGE_LOCAL_ROOT=runtime/production-like/files FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false +FILE_STORAGE_S3_ENDPOINT_TRUSTED=false FILE_ARCHIVE_MAX_ENTRIES=10000 FILE_ARCHIVE_MAX_EXPANDED_BYTES=2147483648 FILE_ARCHIVE_MAX_EXPANSION_RATIO=100 @@ -488,7 +853,11 @@ def _clean(value: str | None) -> str: def _csv(value: str | None) -> tuple[str, ...]: - return tuple(dict.fromkeys(item.strip() for item in str(value or "").split(",") if item.strip())) + return tuple( + dict.fromkeys( + item.strip() for item in str(value or "").split(",") if item.strip() + ) + ) def _truthy(value: str | None) -> bool: diff --git a/src/govoplan_core/core/institutional.py b/src/govoplan_core/core/institutional.py new file mode 100644 index 0000000..c9979de --- /dev/null +++ b/src/govoplan_core/core/institutional.py @@ -0,0 +1,3310 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime +import re +from typing import Literal, Protocol, cast, runtime_checkable +from urllib.parse import urlsplit + +from govoplan_core.core.external_references import ( + ExternalObjectReference, + IntegrationMaturity, + SourceAuthorityMode, +) + + +INSTITUTIONAL_CONTEXT_CONTRACT_VERSION = "1" +CAPABILITY_MANDATE_RESOLVER = "mandates.resolver" +CAPABILITY_SERVICE_DEFINITIONS = "services.definitions" +CAPABILITY_SERVICE_AVAILABILITY = "services.availability" +CAPABILITY_FORM_DEFINITIONS = "forms.definitions" +CAPABILITY_PARTY_RESOLVER = "parties.resolver" +CAPABILITY_DECISION_REGISTRY = "decisions.registry" + +InstitutionalReferenceKind = Literal[ + "institution", + "organization_unit", + "function", + "function_assignment", + "task", + "mandate", + "jurisdiction", + "service", + "form", + "form_submission", + "case", + "party", + "decision", + "work_item", + "workflow", + "approval", + "record", +] +LegalBasisKind = Literal[ + "law", + "regulation", + "statute", + "policy", + "contract", + "delegation", + "mandate", + "organizational_rule", + "exception", + "other", +] +EvidenceKind = Literal[ + "document", + "record", + "event", + "snapshot", + "materialization", + "decision", + "signature", + "external", + "other", +] +PartySubjectKind = Literal["identity", "organization", "group", "external"] +MandateStatus = Literal["draft", "active", "suspended", "replaced", "retired"] +ServicePublicationState = Literal[ + "draft", "published", "suspended", "retired" +] +ServiceAvailabilityRequirementKind = Literal[ + "module", + "capability", + "mandate", + "policy", + "connector", + "maintenance", + "audience", + "configuration", +] +ServiceAvailabilityFailureState = Literal["unavailable", "hidden"] +ServiceLaunchState = Literal["started", "redirect"] +FormPublicationState = Literal["draft", "published", "retired"] +FormValueType = Literal[ + "text", + "multiline_text", + "integer", + "number", + "boolean", + "date", + "datetime", + "email", + "choice", + "multi_choice", + "object", + "list", +] +FormSignatureRequirement = Literal["none", "optional", "required"] +ProcedurePartyStatus = Literal[ + "active", "revoked", "expired", "superseded" +] +DecisionState = Literal[ + "draft", + "proposed", + "approved", + "decided", + "effective", + "corrected", + "revoked", + "superseded", +] +DecisionAssuranceLevel = Literal[ + "human", + "human_reviewed_automation", + "automated_under_mandate", +] +DisclosureState = Literal[ + "not_assessed", + "restricted", + "partly_disclosable", + "disclosable", + "published", +] + +_LEGAL_BASIS_KINDS = frozenset( + { + "law", + "regulation", + "statute", + "policy", + "contract", + "delegation", + "mandate", + "organizational_rule", + "exception", + "other", + } +) +_EVIDENCE_KINDS = frozenset( + { + "document", + "record", + "event", + "snapshot", + "materialization", + "decision", + "signature", + "external", + "other", + } +) +_PARTY_SUBJECT_KINDS = frozenset( + {"identity", "organization", "group", "external"} +) +_MANDATE_STATUSES = frozenset( + {"draft", "active", "suspended", "replaced", "retired"} +) +_MANDATE_TRANSITIONS: Mapping[MandateStatus, frozenset[MandateStatus]] = { + "draft": frozenset({"active", "retired"}), + "active": frozenset({"suspended", "replaced", "retired"}), + "suspended": frozenset({"active", "replaced", "retired"}), + "replaced": frozenset(), + "retired": frozenset(), +} +_SERVICE_PUBLICATION_STATES = frozenset( + {"draft", "published", "suspended", "retired"} +) +_FORM_PUBLICATION_STATES = frozenset({"draft", "published", "retired"}) +_FORM_VALUE_TYPES = frozenset( + { + "text", + "multiline_text", + "integer", + "number", + "boolean", + "date", + "datetime", + "email", + "choice", + "multi_choice", + "object", + "list", + } +) +_FORM_SIGNATURE_REQUIREMENTS = frozenset({"none", "optional", "required"}) +_SERVICE_AVAILABILITY_REQUIREMENT_KINDS = frozenset( + { + "module", + "capability", + "mandate", + "policy", + "connector", + "maintenance", + "audience", + "configuration", + } +) +_PROCEDURE_PARTY_STATUSES = frozenset( + {"active", "revoked", "expired", "superseded"} +) +_PROCEDURE_PARTY_TRANSITIONS: Mapping[ + ProcedurePartyStatus, frozenset[ProcedurePartyStatus] +] = { + "active": frozenset({"active", "revoked", "expired", "superseded"}), + "revoked": frozenset(), + "expired": frozenset(), + "superseded": frozenset(), +} +_DECISION_STATES = frozenset( + { + "draft", + "proposed", + "approved", + "decided", + "effective", + "corrected", + "revoked", + "superseded", + } +) +_DECISION_ASSURANCE_LEVELS = frozenset( + {"human", "human_reviewed_automation", "automated_under_mandate"} +) +_DISCLOSURE_STATES = frozenset( + { + "not_assessed", + "restricted", + "partly_disclosable", + "disclosable", + "published", + } +) +_DECISION_TRANSITIONS: Mapping[DecisionState, frozenset[DecisionState]] = { + "draft": frozenset({"proposed", "revoked"}), + "proposed": frozenset({"approved", "decided", "revoked"}), + "approved": frozenset({"decided", "revoked"}), + "decided": frozenset({"effective", "corrected", "revoked", "superseded"}), + "effective": frozenset({"corrected", "revoked", "superseded"}), + "corrected": frozenset({"effective", "corrected", "revoked", "superseded"}), + "revoked": frozenset(), + "superseded": frozenset(), +} + +_REFERENCE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,254}$") +_MODULE_ID_RE = re.compile(r"^[a-z][a-z0-9_]*$") + + +class InstitutionalContextError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class TemporalRevision: + revision: str + valid_from: datetime | None = None + valid_to: datetime | None = None + recorded_at: datetime | None = None + superseded_at: datetime | None = None + change_reason: str | None = None + + def __post_init__(self) -> None: + _require_identifier(self.revision, "Temporal revision") + for field_name in ( + "valid_from", + "valid_to", + "recorded_at", + "superseded_at", + ): + _require_aware_datetime(getattr(self, field_name), field_name) + if ( + self.valid_from is not None + and self.valid_to is not None + and self.valid_to <= self.valid_from + ): + raise InstitutionalContextError( + "Temporal valid_to must be later than valid_from." + ) + if ( + self.recorded_at is not None + and self.superseded_at is not None + and self.superseded_at < self.recorded_at + ): + raise InstitutionalContextError( + "Temporal superseded_at cannot precede recorded_at." + ) + + def effective_at(self, instant: datetime) -> bool: + _require_aware_datetime(instant, "effective instant") + return ( + (self.valid_from is None or instant >= self.valid_from) + and (self.valid_to is None or instant < self.valid_to) + ) + + def to_dict(self) -> dict[str, object]: + return { + "revision": self.revision, + "valid_from": _datetime_value(self.valid_from), + "valid_to": _datetime_value(self.valid_to), + "recorded_at": _datetime_value(self.recorded_at), + "superseded_at": _datetime_value(self.superseded_at), + "change_reason": self.change_reason, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "TemporalRevision": + return cls( + revision=_required_text(value, "revision"), + valid_from=_optional_datetime(value.get("valid_from")), + valid_to=_optional_datetime(value.get("valid_to")), + recorded_at=_optional_datetime(value.get("recorded_at")), + superseded_at=_optional_datetime(value.get("superseded_at")), + change_reason=_optional_text(value.get("change_reason")), + ) + + +@dataclass(frozen=True, slots=True) +class InstitutionalReference: + kind: InstitutionalReferenceKind + owner_module: str + object_id: str + tenant_id: str + version: str | None = None + valid_at: datetime | None = None + label: str | None = None + + def __post_init__(self) -> None: + if self.kind not in { + "institution", + "organization_unit", + "function", + "function_assignment", + "task", + "mandate", + "jurisdiction", + "service", + "form", + "form_submission", + "case", + "party", + "decision", + "work_item", + "workflow", + "approval", + "record", + }: + raise InstitutionalContextError( + f"Unsupported institutional reference kind: {self.kind!r}." + ) + if not _MODULE_ID_RE.fullmatch(self.owner_module): + raise InstitutionalContextError( + f"Invalid institutional reference owner module: {self.owner_module!r}." + ) + _require_identifier(self.object_id, "Institutional object id") + _require_identifier(self.tenant_id, "Institutional tenant id") + _require_aware_datetime(self.valid_at, "institutional valid_at") + + def to_dict(self, *, disclose_label: bool = False) -> dict[str, object]: + return { + "kind": self.kind, + "owner_module": self.owner_module, + "object_id": self.object_id, + "tenant_id": self.tenant_id, + "version": self.version, + "valid_at": _datetime_value(self.valid_at), + "label": self.label if disclose_label else None, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "InstitutionalReference": + return cls( + kind=cast(InstitutionalReferenceKind, _required_text(value, "kind")), + owner_module=_required_text(value, "owner_module"), + object_id=_required_text(value, "object_id"), + tenant_id=_required_text(value, "tenant_id"), + version=_optional_text(value.get("version")), + valid_at=_optional_datetime(value.get("valid_at")), + label=_optional_text(value.get("label")), + ) + + +@dataclass(frozen=True, slots=True) +class LegalBasisReference: + kind: LegalBasisKind + authority: str + reference: str + version: str + effective_from: datetime | None = None + effective_to: datetime | None = None + title: str | None = None + inspection_url: str | None = None + + def __post_init__(self) -> None: + if self.kind not in _LEGAL_BASIS_KINDS: + raise InstitutionalContextError( + f"Unsupported legal basis kind: {self.kind!r}." + ) + _require_text(self.authority, "Legal basis authority") + _require_identifier(self.reference, "Legal basis reference") + _require_identifier(self.version, "Legal basis version") + _require_aware_datetime(self.effective_from, "legal basis effective_from") + _require_aware_datetime(self.effective_to, "legal basis effective_to") + if ( + self.effective_from is not None + and self.effective_to is not None + and self.effective_to <= self.effective_from + ): + raise InstitutionalContextError( + "Legal basis effective_to must be later than effective_from." + ) + _validate_inspection_url(self.inspection_url) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "kind": self.kind, + "authority": self.authority, + "reference": self.reference, + "version": self.version, + "effective_from": _datetime_value(self.effective_from), + "effective_to": _datetime_value(self.effective_to), + "title": self.title if include_inspection else None, + "inspection_url": self.inspection_url if include_inspection else None, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "LegalBasisReference": + return cls( + kind=cast(LegalBasisKind, _required_text(value, "kind")), + authority=_required_text(value, "authority"), + reference=_required_text(value, "reference"), + version=_required_text(value, "version"), + effective_from=_optional_datetime(value.get("effective_from")), + effective_to=_optional_datetime(value.get("effective_to")), + title=_optional_text(value.get("title")), + inspection_url=_optional_text(value.get("inspection_url")), + ) + + +@dataclass(frozen=True, slots=True) +class EvidenceReference: + kind: EvidenceKind + owner_module: str + evidence_id: str + tenant_id: str + version: str | None = None + checksum: str | None = None + source_ref: str | None = None + derived_from: tuple[str, ...] = () + responsible_actor_ref: str | None = None + captured_at: datetime | None = None + inspection_url: str | None = None + + def __post_init__(self) -> None: + if self.kind not in _EVIDENCE_KINDS: + raise InstitutionalContextError( + f"Unsupported evidence kind: {self.kind!r}." + ) + if not _MODULE_ID_RE.fullmatch(self.owner_module): + raise InstitutionalContextError( + f"Invalid evidence owner module: {self.owner_module!r}." + ) + _require_identifier(self.evidence_id, "Evidence id") + _require_identifier(self.tenant_id, "Evidence tenant id") + _require_aware_datetime(self.captured_at, "evidence captured_at") + _validate_text_tuple(self.derived_from, "Evidence derivation references") + _validate_inspection_url(self.inspection_url) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "kind": self.kind, + "owner_module": self.owner_module, + "evidence_id": self.evidence_id, + "tenant_id": self.tenant_id, + "version": self.version, + "checksum": self.checksum, + "source_ref": self.source_ref, + "derived_from": list(self.derived_from), + "responsible_actor_ref": self.responsible_actor_ref, + "captured_at": _datetime_value(self.captured_at), + "inspection_url": self.inspection_url if include_inspection else None, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "EvidenceReference": + return cls( + kind=cast(EvidenceKind, _required_text(value, "kind")), + owner_module=_required_text(value, "owner_module"), + evidence_id=_required_text(value, "evidence_id"), + tenant_id=_required_text(value, "tenant_id"), + version=_optional_text(value.get("version")), + checksum=_optional_text(value.get("checksum")), + source_ref=_optional_text(value.get("source_ref")), + derived_from=_text_tuple(value.get("derived_from")), + responsible_actor_ref=_optional_text(value.get("responsible_actor_ref")), + captured_at=_optional_datetime(value.get("captured_at")), + inspection_url=_optional_text(value.get("inspection_url")), + ) + + +@dataclass(frozen=True, slots=True) +class InformationGovernanceReference: + classification: str + purposes: tuple[str, ...] + legal_basis_refs: tuple[str, ...] = () + retention_policy_ref: str | None = None + hold_refs: tuple[str, ...] = () + minimization_profile: str | None = None + disclosure_state: DisclosureState = "not_assessed" + + def __post_init__(self) -> None: + if self.disclosure_state not in _DISCLOSURE_STATES: + raise InstitutionalContextError( + f"Unsupported disclosure state: {self.disclosure_state!r}." + ) + _require_text(self.classification, "Information classification") + _validate_text_tuple(self.purposes, "Information purposes", require=True) + _validate_text_tuple(self.legal_basis_refs, "Information legal basis refs") + _validate_text_tuple(self.hold_refs, "Information hold refs") + + def to_dict(self) -> dict[str, object]: + return { + "classification": self.classification, + "purposes": list(self.purposes), + "legal_basis_refs": list(self.legal_basis_refs), + "retention_policy_ref": self.retention_policy_ref, + "hold_refs": list(self.hold_refs), + "minimization_profile": self.minimization_profile, + "disclosure_state": self.disclosure_state, + } + + @classmethod + def from_mapping( + cls, value: Mapping[str, object] + ) -> "InformationGovernanceReference": + return cls( + classification=_required_text(value, "classification"), + purposes=_text_tuple(value.get("purposes")), + legal_basis_refs=_text_tuple(value.get("legal_basis_refs")), + retention_policy_ref=_optional_text(value.get("retention_policy_ref")), + hold_refs=_text_tuple(value.get("hold_refs")), + minimization_profile=_optional_text(value.get("minimization_profile")), + disclosure_state=cast( + DisclosureState, + str(value.get("disclosure_state") or "not_assessed"), + ), + ) + + +@dataclass(frozen=True, slots=True) +class GeoReference: + reference_id: str + crs: str + latitude: float | None = None + longitude: float | None = None + geometry_ref: str | None = None + administrative_area_ref: str | None = None + address_ref: str | None = None + location_ref: str | None = None + source: str | None = None + accuracy_meters: float | None = None + valid_from: datetime | None = None + valid_to: datetime | None = None + external_gis: ExternalObjectReference | None = None + + def __post_init__(self) -> None: + _require_identifier(self.reference_id, "Geo reference id") + _require_text(self.crs, "Geo CRS") + if (self.latitude is None) != (self.longitude is None): + raise InstitutionalContextError( + "Geo latitude and longitude must be supplied together." + ) + if self.latitude is not None and not -90 <= self.latitude <= 90: + raise InstitutionalContextError("Geo latitude is outside -90..90.") + if self.longitude is not None and not -180 <= self.longitude <= 180: + raise InstitutionalContextError("Geo longitude is outside -180..180.") + if self.accuracy_meters is not None and self.accuracy_meters < 0: + raise InstitutionalContextError("Geo accuracy cannot be negative.") + _require_aware_datetime(self.valid_from, "geo valid_from") + _require_aware_datetime(self.valid_to, "geo valid_to") + if ( + self.valid_from is not None + and self.valid_to is not None + and self.valid_to <= self.valid_from + ): + raise InstitutionalContextError( + "Geo valid_to must be later than valid_from." + ) + + def to_dict(self) -> dict[str, object]: + return { + "reference_id": self.reference_id, + "crs": self.crs, + "latitude": self.latitude, + "longitude": self.longitude, + "geometry_ref": self.geometry_ref, + "administrative_area_ref": self.administrative_area_ref, + "address_ref": self.address_ref, + "location_ref": self.location_ref, + "source": self.source, + "accuracy_meters": self.accuracy_meters, + "valid_from": _datetime_value(self.valid_from), + "valid_to": _datetime_value(self.valid_to), + "external_gis": ( + self.external_gis.to_dict() if self.external_gis is not None else None + ), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "GeoReference": + external = value.get("external_gis") + return cls( + reference_id=_required_text(value, "reference_id"), + crs=_required_text(value, "crs"), + latitude=_optional_float(value.get("latitude")), + longitude=_optional_float(value.get("longitude")), + geometry_ref=_optional_text(value.get("geometry_ref")), + administrative_area_ref=_optional_text( + value.get("administrative_area_ref") + ), + address_ref=_optional_text(value.get("address_ref")), + location_ref=_optional_text(value.get("location_ref")), + source=_optional_text(value.get("source")), + accuracy_meters=_optional_float(value.get("accuracy_meters")), + valid_from=_optional_datetime(value.get("valid_from")), + valid_to=_optional_datetime(value.get("valid_to")), + external_gis=( + _external_reference_from_mapping(external) + if isinstance(external, Mapping) + else None + ), + ) + + +@dataclass(frozen=True, slots=True) +class ActorRepresentationReference: + tenant_id: str + account_id: str | None = None + identity_id: str | None = None + service_account_id: str | None = None + represented_account_id: str | None = None + represented_function_ref: InstitutionalReference | None = None + represented_party_ref: InstitutionalReference | None = None + function_assignment_ref: InstitutionalReference | None = None + delegation_ref: str | None = None + power_ref: str | None = None + mandate_ref: InstitutionalReference | None = None + + def __post_init__(self) -> None: + _require_identifier(self.tenant_id, "Actor tenant id") + if bool(self.account_id) == bool(self.service_account_id): + raise InstitutionalContextError( + "Actor representation requires exactly one real account or service account." + ) + _validate_reference_kind( + self.represented_function_ref, "function", "represented function" + ) + _validate_reference_kind( + self.represented_party_ref, "party", "represented party" + ) + _validate_reference_kind( + self.function_assignment_ref, + "function_assignment", + "function assignment", + ) + _validate_reference_kind(self.mandate_ref, "mandate", "actor mandate") + _validate_reference_tenants( + self.tenant_id, + ( + self.represented_function_ref, + self.represented_party_ref, + self.function_assignment_ref, + self.mandate_ref, + ), + ) + + def to_dict(self) -> dict[str, object]: + return { + "tenant_id": self.tenant_id, + "account_id": self.account_id, + "identity_id": self.identity_id, + "service_account_id": self.service_account_id, + "represented_account_id": self.represented_account_id, + "represented_function_ref": _reference_value( + self.represented_function_ref + ), + "represented_party_ref": _reference_value(self.represented_party_ref), + "function_assignment_ref": _reference_value( + self.function_assignment_ref + ), + "delegation_ref": self.delegation_ref, + "power_ref": self.power_ref, + "mandate_ref": _reference_value(self.mandate_ref), + } + + @classmethod + def from_mapping( + cls, value: Mapping[str, object] + ) -> "ActorRepresentationReference": + return cls( + tenant_id=_required_text(value, "tenant_id"), + account_id=_optional_text(value.get("account_id")), + identity_id=_optional_text(value.get("identity_id")), + service_account_id=_optional_text(value.get("service_account_id")), + represented_account_id=_optional_text( + value.get("represented_account_id") + ), + represented_function_ref=_optional_institutional_reference( + value.get("represented_function_ref") + ), + represented_party_ref=_optional_institutional_reference( + value.get("represented_party_ref") + ), + function_assignment_ref=_optional_institutional_reference( + value.get("function_assignment_ref") + ), + delegation_ref=_optional_text(value.get("delegation_ref")), + power_ref=_optional_text(value.get("power_ref")), + mandate_ref=_optional_institutional_reference( + value.get("mandate_ref") + ), + ) + + +@dataclass(frozen=True, slots=True) +class ExternalSourceReference: + reference: ExternalObjectReference + authority_mode: SourceAuthorityMode + maturity: IntegrationMaturity + freshness_state: str + health_state: str + conflict_state: str = "none" + + def __post_init__(self) -> None: + if self.reference.authority_mode != self.authority_mode: + raise InstitutionalContextError( + "External source authority mode must match its object reference." + ) + if self.reference.maturity != self.maturity: + raise InstitutionalContextError( + "External source maturity must match its object reference." + ) + _require_text(self.freshness_state, "External source freshness state") + _require_text(self.health_state, "External source health state") + _require_text(self.conflict_state, "External source conflict state") + + def to_dict(self) -> dict[str, object]: + return { + "reference": self.reference.to_dict(), + "authority_mode": self.authority_mode, + "maturity": self.maturity, + "freshness_state": self.freshness_state, + "health_state": self.health_state, + "conflict_state": self.conflict_state, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "ExternalSourceReference": + reference = value.get("reference") + if not isinstance(reference, Mapping): + raise InstitutionalContextError( + "External source reference payload is required." + ) + return cls( + reference=_external_reference_from_mapping(reference), + authority_mode=cast( + SourceAuthorityMode, _required_text(value, "authority_mode") + ), + maturity=cast(IntegrationMaturity, _required_text(value, "maturity")), + freshness_state=_required_text(value, "freshness_state"), + health_state=_required_text(value, "health_state"), + conflict_state=str(value.get("conflict_state") or "none"), + ) + + +@dataclass(frozen=True, slots=True) +class PresentationReference: + language: str | None = None + accessibility_profile: str | None = None + channel: str | None = None + explanation_ref: str | None = None + availability_ref: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "language": self.language, + "accessibility_profile": self.accessibility_profile, + "channel": self.channel, + "explanation_ref": self.explanation_ref, + "availability_ref": self.availability_ref, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "PresentationReference": + return cls( + language=_optional_text(value.get("language")), + accessibility_profile=_optional_text( + value.get("accessibility_profile") + ), + channel=_optional_text(value.get("channel")), + explanation_ref=_optional_text(value.get("explanation_ref")), + availability_ref=_optional_text(value.get("availability_ref")), + ) + + +@dataclass(frozen=True, slots=True) +class GovernedContextEnvelope: + tenant_id: str + temporal: TemporalRevision + actor: ActorRepresentationReference | None = None + institution_ref: InstitutionalReference | None = None + organization_unit_ref: InstitutionalReference | None = None + function_ref: InstitutionalReference | None = None + task_ref: InstitutionalReference | None = None + mandate_ref: InstitutionalReference | None = None + jurisdiction_refs: tuple[InstitutionalReference, ...] = () + service_ref: InstitutionalReference | None = None + case_ref: InstitutionalReference | None = None + party_refs: tuple[InstitutionalReference, ...] = () + work_item_ref: InstitutionalReference | None = None + workflow_ref: InstitutionalReference | None = None + approval_refs: tuple[InstitutionalReference, ...] = () + decision_ref: InstitutionalReference | None = None + record_refs: tuple[InstitutionalReference, ...] = () + legal_bases: tuple[LegalBasisReference, ...] = () + evidence: tuple[EvidenceReference, ...] = () + information_governance: InformationGovernanceReference | None = None + external_sources: tuple[ExternalSourceReference, ...] = () + presentation: PresentationReference | None = None + contract_version: str = INSTITUTIONAL_CONTEXT_CONTRACT_VERSION + + def __post_init__(self) -> None: + if self.contract_version != INSTITUTIONAL_CONTEXT_CONTRACT_VERSION: + raise InstitutionalContextError( + "Unsupported institutional context contract version." + ) + _require_identifier(self.tenant_id, "Institutional context tenant id") + _validate_reference_kind( + self.institution_ref, "institution", "institution reference" + ) + _validate_reference_kind( + self.organization_unit_ref, + "organization_unit", + "organization unit reference", + ) + _validate_reference_kind(self.function_ref, "function", "function reference") + _validate_reference_kind(self.task_ref, "task", "task reference") + _validate_reference_kind(self.mandate_ref, "mandate", "mandate reference") + _validate_reference_kind(self.service_ref, "service", "service reference") + _validate_reference_kind(self.case_ref, "case", "case reference") + _validate_reference_kind(self.decision_ref, "decision", "decision reference") + for item in self.jurisdiction_refs: + _validate_reference_kind(item, "jurisdiction", "jurisdiction reference") + for item in self.party_refs: + _validate_reference_kind(item, "party", "party reference") + _validate_reference_kind( + self.work_item_ref, "work_item", "work item reference" + ) + _validate_reference_kind(self.workflow_ref, "workflow", "workflow reference") + for item in self.approval_refs: + _validate_reference_kind(item, "approval", "approval reference") + for item in self.record_refs: + _validate_reference_kind(item, "record", "record reference") + references = ( + self.institution_ref, + self.organization_unit_ref, + self.function_ref, + self.task_ref, + self.mandate_ref, + *self.jurisdiction_refs, + self.service_ref, + self.case_ref, + *self.party_refs, + self.work_item_ref, + self.workflow_ref, + *self.approval_refs, + self.decision_ref, + *self.record_refs, + ) + _validate_reference_tenants(self.tenant_id, references) + if self.actor is not None and self.actor.tenant_id != self.tenant_id: + raise InstitutionalContextError( + "Actor representation belongs to a different tenant." + ) + for item in self.evidence: + if item.tenant_id != self.tenant_id: + raise InstitutionalContextError( + "Evidence reference belongs to a different tenant." + ) + + def effective_at(self, instant: datetime) -> bool: + if not self.temporal.effective_at(instant): + return False + return all( + item.valid_at is None or item.valid_at == instant + for item in ( + self.mandate_ref, + self.service_ref, + self.case_ref, + self.decision_ref, + ) + if item is not None + ) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "contract_version": self.contract_version, + "tenant_id": self.tenant_id, + "temporal": self.temporal.to_dict(), + "actor": self.actor.to_dict() if self.actor else None, + "institution_ref": _reference_value(self.institution_ref), + "organization_unit_ref": _reference_value( + self.organization_unit_ref + ), + "function_ref": _reference_value(self.function_ref), + "task_ref": _reference_value(self.task_ref), + "mandate_ref": _reference_value(self.mandate_ref), + "jurisdiction_refs": [ + item.to_dict() for item in self.jurisdiction_refs + ], + "service_ref": _reference_value(self.service_ref), + "case_ref": _reference_value(self.case_ref), + "party_refs": [item.to_dict() for item in self.party_refs], + "work_item_ref": _reference_value(self.work_item_ref), + "workflow_ref": _reference_value(self.workflow_ref), + "approval_refs": [item.to_dict() for item in self.approval_refs], + "decision_ref": _reference_value(self.decision_ref), + "record_refs": [item.to_dict() for item in self.record_refs], + "legal_bases": [ + item.to_dict(include_inspection=include_inspection) + for item in self.legal_bases + ], + "evidence": [ + item.to_dict(include_inspection=include_inspection) + for item in self.evidence + ], + "information_governance": ( + self.information_governance.to_dict() + if self.information_governance + else None + ), + "external_sources": [item.to_dict() for item in self.external_sources], + "presentation": self.presentation.to_dict() if self.presentation else None, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "GovernedContextEnvelope": + temporal = value.get("temporal") + if not isinstance(temporal, Mapping): + raise InstitutionalContextError( + "Institutional context temporal revision is required." + ) + return cls( + contract_version=str( + value.get("contract_version") + or INSTITUTIONAL_CONTEXT_CONTRACT_VERSION + ), + tenant_id=_required_text(value, "tenant_id"), + temporal=TemporalRevision.from_mapping(temporal), + actor=_optional_actor(value.get("actor")), + institution_ref=_optional_institutional_reference( + value.get("institution_ref") + ), + organization_unit_ref=_optional_institutional_reference( + value.get("organization_unit_ref") + ), + function_ref=_optional_institutional_reference( + value.get("function_ref") + ), + task_ref=_optional_institutional_reference(value.get("task_ref")), + mandate_ref=_optional_institutional_reference( + value.get("mandate_ref") + ), + jurisdiction_refs=_institutional_references( + value.get("jurisdiction_refs") + ), + service_ref=_optional_institutional_reference( + value.get("service_ref") + ), + case_ref=_optional_institutional_reference(value.get("case_ref")), + party_refs=_institutional_references(value.get("party_refs")), + work_item_ref=_optional_institutional_reference( + value.get("work_item_ref") + ), + workflow_ref=_optional_institutional_reference( + value.get("workflow_ref") + ), + approval_refs=_institutional_references(value.get("approval_refs")), + decision_ref=_optional_institutional_reference( + value.get("decision_ref") + ), + record_refs=_institutional_references(value.get("record_refs")), + legal_bases=tuple( + LegalBasisReference.from_mapping(item) + for item in _mapping_sequence(value.get("legal_bases")) + ), + evidence=tuple( + EvidenceReference.from_mapping(item) + for item in _mapping_sequence(value.get("evidence")) + ), + information_governance=_optional_governance( + value.get("information_governance") + ), + external_sources=tuple( + ExternalSourceReference.from_mapping(item) + for item in _mapping_sequence(value.get("external_sources")) + ), + presentation=_optional_presentation(value.get("presentation")), + ) + + +@dataclass(frozen=True, slots=True) +class MandateDefinition: + reference: InstitutionalReference + temporal: TemporalRevision + task_types: tuple[str, ...] + authority_types: tuple[str, ...] + organization_unit_refs: tuple[InstitutionalReference, ...] = () + function_refs: tuple[InstitutionalReference, ...] = () + jurisdiction_refs: tuple[InstitutionalReference, ...] = () + subject_types: tuple[str, ...] = () + authority_ceiling: str | None = None + legal_bases: tuple[LegalBasisReference, ...] = () + evidence: tuple[EvidenceReference, ...] = () + status: MandateStatus = "active" + replacement_ref: InstitutionalReference | None = None + suspension_reason: str | None = None + conflict_refs: tuple[str, ...] = () + + def __post_init__(self) -> None: + _validate_reference_kind(self.reference, "mandate", "mandate definition") + if self.status not in _MANDATE_STATUSES: + raise InstitutionalContextError( + f"Unsupported mandate status: {self.status!r}." + ) + _validate_text_tuple(self.task_types, "Mandate task types", require=True) + _validate_text_tuple( + self.authority_types, "Mandate authority types", require=True + ) + for item in self.organization_unit_refs: + _validate_reference_kind(item, "organization_unit", "mandate unit") + for item in self.function_refs: + _validate_reference_kind(item, "function", "mandate function") + for item in self.jurisdiction_refs: + _validate_reference_kind(item, "jurisdiction", "mandate jurisdiction") + _validate_reference_kind( + self.replacement_ref, + "mandate", + "replaced mandate reference", + ) + _validate_text_tuple(self.conflict_refs, "Mandate conflict references") + if self.status == "suspended" and not str( + self.suspension_reason or "" + ).strip(): + raise InstitutionalContextError( + "Suspended mandates require a suspension reason." + ) + if self.status == "replaced" and self.replacement_ref is None: + raise InstitutionalContextError( + "Replaced mandates require the prior mandate reference." + ) + _validate_reference_tenants( + self.reference.tenant_id, + ( + *self.organization_unit_refs, + *self.function_refs, + *self.jurisdiction_refs, + self.replacement_ref, + ), + ) + for item in self.evidence: + if item.tenant_id != self.reference.tenant_id: + raise InstitutionalContextError( + "Mandate evidence belongs to a different tenant." + ) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "reference": self.reference.to_dict(), + "temporal": self.temporal.to_dict(), + "task_types": list(self.task_types), + "authority_types": list(self.authority_types), + "organization_unit_refs": [ + item.to_dict() for item in self.organization_unit_refs + ], + "function_refs": [item.to_dict() for item in self.function_refs], + "jurisdiction_refs": [ + item.to_dict() for item in self.jurisdiction_refs + ], + "subject_types": list(self.subject_types), + "authority_ceiling": self.authority_ceiling, + "legal_bases": [ + item.to_dict(include_inspection=include_inspection) + for item in self.legal_bases + ], + "evidence": [ + item.to_dict(include_inspection=include_inspection) + for item in self.evidence + ], + "status": self.status, + "replacement_ref": _reference_value(self.replacement_ref), + "suspension_reason": self.suspension_reason, + "conflict_refs": list(self.conflict_refs), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "MandateDefinition": + reference = _required_mapping(value, "reference") + temporal = _required_mapping(value, "temporal") + return cls( + reference=InstitutionalReference.from_mapping(reference), + temporal=TemporalRevision.from_mapping(temporal), + task_types=_text_tuple(value.get("task_types")), + authority_types=_text_tuple(value.get("authority_types")), + organization_unit_refs=_institutional_references( + value.get("organization_unit_refs") + ), + function_refs=_institutional_references(value.get("function_refs")), + jurisdiction_refs=_institutional_references( + value.get("jurisdiction_refs") + ), + subject_types=_text_tuple(value.get("subject_types")), + authority_ceiling=_optional_text(value.get("authority_ceiling")), + legal_bases=tuple( + LegalBasisReference.from_mapping(item) + for item in _mapping_sequence(value.get("legal_bases")) + ), + evidence=tuple( + EvidenceReference.from_mapping(item) + for item in _mapping_sequence(value.get("evidence")) + ), + status=cast(MandateStatus, str(value.get("status") or "active")), + replacement_ref=_optional_institutional_reference( + value.get("replacement_ref") + ), + suspension_reason=_optional_text(value.get("suspension_reason")), + conflict_refs=_text_tuple(value.get("conflict_refs")), + ) + + +def revise_mandate_definition( + current: MandateDefinition, + *, + expected_revision: str, + temporal: TemporalRevision, + status: MandateStatus, + replacement_ref: InstitutionalReference | None = None, + suspension_reason: str | None = None, + conflict_refs: Sequence[str] | None = None, +) -> MandateDefinition: + """Create an immutable, OCC-guarded Mandate lifecycle revision.""" + + if current.temporal.revision != expected_revision: + raise InstitutionalContextError( + "Mandate revision conflict: the expected revision is stale." + ) + if temporal.revision == current.temporal.revision: + raise InstitutionalContextError( + "A Mandate revision requires a new revision identifier." + ) + if temporal.recorded_at is None or not str( + temporal.change_reason or "" + ).strip(): + raise InstitutionalContextError( + "A Mandate revision requires recorded_at and change_reason." + ) + if status not in _MANDATE_TRANSITIONS[current.status]: + raise InstitutionalContextError( + f"Mandate transition {current.status!r} to {status!r} is not allowed." + ) + if status == "replaced" and replacement_ref is None: + raise InstitutionalContextError( + "Replacing a Mandate requires the replacement Mandate reference." + ) + + return replace( + current, + reference=replace( + current.reference, + version=temporal.revision, + valid_at=temporal.valid_from or current.reference.valid_at, + ), + temporal=temporal, + status=status, + replacement_ref=replacement_ref if status == "replaced" else None, + suspension_reason=( + suspension_reason if status == "suspended" else None + ), + conflict_refs=( + tuple(conflict_refs) + if conflict_refs is not None + else current.conflict_refs + ), + ) + + +@dataclass(frozen=True, slots=True) +class MandateResolutionRequest: + tenant_id: str + effective_at: datetime + task_type: str + authority_type: str + organization_unit_ref: InstitutionalReference | None = None + function_ref: InstitutionalReference | None = None + jurisdiction_refs: tuple[InstitutionalReference, ...] = () + subject_type: str | None = None + geo_ref: GeoReference | None = None + + def __post_init__(self) -> None: + _require_identifier(self.tenant_id, "Mandate request tenant id") + _require_aware_datetime(self.effective_at, "mandate request effective_at") + _require_text(self.task_type, "Mandate request task type") + _require_text(self.authority_type, "Mandate request authority type") + _validate_reference_kind( + self.organization_unit_ref, + "organization_unit", + "mandate request organization unit", + ) + _validate_reference_kind( + self.function_ref, + "function", + "mandate request function", + ) + for item in self.jurisdiction_refs: + _validate_reference_kind( + item, + "jurisdiction", + "mandate request jurisdiction", + ) + _validate_reference_tenants( + self.tenant_id, + ( + self.organization_unit_ref, + self.function_ref, + *self.jurisdiction_refs, + ), + ) + + def to_dict(self) -> dict[str, object]: + return { + "tenant_id": self.tenant_id, + "effective_at": self.effective_at.isoformat(), + "task_type": self.task_type, + "authority_type": self.authority_type, + "organization_unit_ref": _reference_value( + self.organization_unit_ref + ), + "function_ref": _reference_value(self.function_ref), + "jurisdiction_refs": [ + item.to_dict() for item in self.jurisdiction_refs + ], + "subject_type": self.subject_type, + "geo_ref": self.geo_ref.to_dict() if self.geo_ref else None, + } + + @classmethod + def from_mapping( + cls, value: Mapping[str, object] + ) -> "MandateResolutionRequest": + geo = value.get("geo_ref") + return cls( + tenant_id=_required_text(value, "tenant_id"), + effective_at=_required_datetime(value.get("effective_at")), + task_type=_required_text(value, "task_type"), + authority_type=_required_text(value, "authority_type"), + organization_unit_ref=_optional_institutional_reference( + value.get("organization_unit_ref") + ), + function_ref=_optional_institutional_reference( + value.get("function_ref") + ), + jurisdiction_refs=_institutional_references( + value.get("jurisdiction_refs") + ), + subject_type=_optional_text(value.get("subject_type")), + geo_ref=GeoReference.from_mapping(geo) if isinstance(geo, Mapping) else None, + ) + + +@dataclass(frozen=True, slots=True) +class MandateResolution: + competent: bool + mandates: tuple[MandateDefinition, ...] = () + explanation: str | None = None + conflict_refs: tuple[str, ...] = () + evidence: tuple[EvidenceReference, ...] = () + + def __post_init__(self) -> None: + _validate_text_tuple( + self.conflict_refs, + "Mandate resolution conflict references", + ) + if self.competent and not self.mandates: + raise InstitutionalContextError( + "A competent mandate resolution requires at least one mandate." + ) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "competent": self.competent, + "mandates": [ + item.to_dict(include_inspection=include_inspection) + for item in self.mandates + ], + "explanation": self.explanation, + "conflict_refs": list(self.conflict_refs), + "evidence": [ + item.to_dict(include_inspection=include_inspection) + for item in self.evidence + ], + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "MandateResolution": + return cls( + competent=_boolean_value(value, "competent", default=False), + mandates=tuple( + MandateDefinition.from_mapping(item) + for item in _mapping_sequence(value.get("mandates")) + ), + explanation=_optional_text(value.get("explanation")), + conflict_refs=_text_tuple(value.get("conflict_refs")), + evidence=tuple( + EvidenceReference.from_mapping(item) + for item in _mapping_sequence(value.get("evidence")) + ), + ) + + +@runtime_checkable +class MandateResolver(Protocol): + def resolve_mandate( + self, + session: object, + principal: object, + *, + request: MandateResolutionRequest, + ) -> MandateResolution: ... + + +def resolve_mandate_candidates( + request: MandateResolutionRequest, + candidates: Sequence[MandateDefinition], +) -> MandateResolution: + """Resolve effective competence deterministically from governed candidates.""" + + ordered = tuple( + sorted( + candidates, + key=lambda item: ( + item.reference.object_id, + item.temporal.revision, + ), + ) + ) + for mandate in ordered: + if mandate.reference.tenant_id != request.tenant_id: + raise InstitutionalContextError( + "Mandate resolution candidates cannot cross tenants." + ) + effective = tuple( + mandate + for mandate in ordered + if _mandate_matches_request(mandate, request) + ) + candidate_conflicts = tuple( + dict.fromkeys( + conflict + for mandate in effective + for conflict in mandate.conflict_refs + ) + ) + if candidate_conflicts: + return MandateResolution( + competent=False, + mandates=effective, + explanation="Matching mandates have unresolved conflicts.", + conflict_refs=candidate_conflicts, + evidence=_mandate_evidence(effective), + ) + if len(effective) != 1: + conflicts = ( + tuple( + dict.fromkeys( + f"mandate:{item.reference.object_id}@{item.temporal.revision}" + for item in effective + ) + ) + if len(effective) > 1 + else () + ) + return MandateResolution( + competent=False, + mandates=effective, + explanation=( + "Mandate resolution requires exactly one effective active mandate." + ), + conflict_refs=conflicts, + evidence=_mandate_evidence(effective), + ) + return MandateResolution( + competent=True, + mandates=effective, + explanation=( + "Exactly one active mandate covers the effective time, task, " + "authority, organization, function, jurisdiction, and subject." + ), + evidence=_mandate_evidence(effective), + ) + + +def _mandate_matches_request( + mandate: MandateDefinition, + request: MandateResolutionRequest, +) -> bool: + if mandate.status != "active" or not mandate.temporal.effective_at( + request.effective_at + ): + return False + if ( + request.task_type not in mandate.task_types + or request.authority_type not in mandate.authority_types + ): + return False + if mandate.organization_unit_refs and not _reference_is_allowed( + request.organization_unit_ref, + mandate.organization_unit_refs, + ): + return False + if mandate.function_refs and not _reference_is_allowed( + request.function_ref, + mandate.function_refs, + ): + return False + requested_jurisdictions = { + _stable_reference_key(item) for item in request.jurisdiction_refs + } + if mandate.jurisdiction_refs and not { + _stable_reference_key(item) for item in mandate.jurisdiction_refs + }.issubset(requested_jurisdictions): + return False + return not ( + mandate.subject_types + and request.subject_type not in mandate.subject_types + ) + + +def _mandate_evidence( + mandates: Sequence[MandateDefinition], +) -> tuple[EvidenceReference, ...]: + evidence_by_key: dict[tuple[str, str, str, str, str | None], EvidenceReference] = {} + for mandate in mandates: + for item in mandate.evidence: + key = ( + item.kind, + item.owner_module, + item.evidence_id, + item.tenant_id, + item.version, + ) + evidence_by_key.setdefault(key, item) + return tuple(evidence_by_key.values()) + + +@dataclass(frozen=True, slots=True) +class ServiceBinding: + kind: str + reference: str + required: bool = True + + def __post_init__(self) -> None: + _require_text(self.kind, "Service binding kind") + _require_identifier(self.reference, "Service binding reference") + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "reference": self.reference, + "required": self.required, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "ServiceBinding": + return cls( + kind=_required_text(value, "kind"), + reference=_required_text(value, "reference"), + required=_boolean_value(value, "required", default=True), + ) + + +@dataclass(frozen=True, slots=True) +class ServiceAvailabilityRequirement: + kind: ServiceAvailabilityRequirementKind + reference: str + failure_state: ServiceAvailabilityFailureState = "unavailable" + explanation_ref: str | None = None + + def __post_init__(self) -> None: + if self.kind not in _SERVICE_AVAILABILITY_REQUIREMENT_KINDS: + raise InstitutionalContextError( + f"Unsupported Service availability requirement: {self.kind!r}." + ) + _require_identifier(self.reference, "Service availability reference") + if self.explanation_ref is not None: + _require_identifier( + self.explanation_ref, + "Service availability explanation reference", + ) + if self.failure_state not in {"unavailable", "hidden"}: + raise InstitutionalContextError( + f"Unsupported Service availability failure state: {self.failure_state!r}." + ) + + @property + def key(self) -> str: + return f"{self.kind}:{self.reference}" + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "reference": self.reference, + "failure_state": self.failure_state, + "explanation_ref": self.explanation_ref, + } + + @classmethod + def from_mapping( + cls, + value: Mapping[str, object], + ) -> "ServiceAvailabilityRequirement": + return cls( + kind=cast( + ServiceAvailabilityRequirementKind, + _required_text(value, "kind"), + ), + reference=_required_text(value, "reference"), + failure_state=cast( + ServiceAvailabilityFailureState, + str(value.get("failure_state") or "unavailable"), + ), + explanation_ref=_optional_text(value.get("explanation_ref")), + ) + + +@dataclass(frozen=True, slots=True) +class ServiceAvailabilityAssessment: + requirement_states: Mapping[str, bool] + reason_codes: tuple[str, ...] = () + evidence: tuple[EvidenceReference, ...] = () + + def __post_init__(self) -> None: + for key, value in self.requirement_states.items(): + _require_text(key, "Service availability assessment key") + if not isinstance(value, bool): + raise InstitutionalContextError( + "Service availability assessment states must be booleans." + ) + for code in self.reason_codes: + _require_identifier(code, "Service availability reason code") + + +@dataclass(frozen=True, slots=True) +class ServiceDefinition: + reference: InstitutionalReference + key: str + temporal: TemporalRevision + title: str + audience: tuple[str, ...] + prerequisites: tuple[str, ...] = () + legal_bases: tuple[LegalBasisReference, ...] = () + required_evidence_types: tuple[str, ...] = () + fee_refs: tuple[str, ...] = () + deadline_refs: tuple[str, ...] = () + channels: tuple[str, ...] = () + responsible_organization_ref: InstitutionalReference | None = None + responsible_function_ref: InstitutionalReference | None = None + mandate_ref: InstitutionalReference | None = None + jurisdiction_refs: tuple[InstitutionalReference, ...] = () + bindings: tuple[ServiceBinding, ...] = () + availability_requirements: tuple[ServiceAvailabilityRequirement, ...] = () + remedy_refs: tuple[str, ...] = () + service_level_refs: tuple[str, ...] = () + publication_state: ServicePublicationState = "draft" + availability_explanation_ref: str | None = None + derived_from_ref: InstitutionalReference | None = None + + def __post_init__(self) -> None: + _validate_reference_kind(self.reference, "service", "service definition") + _require_identifier(self.key, "Service key") + _require_text(self.title, "Service title") + if self.publication_state not in _SERVICE_PUBLICATION_STATES: + raise InstitutionalContextError( + f"Unsupported service publication state: {self.publication_state!r}." + ) + _validate_text_tuple(self.audience, "Service audience", require=True) + for label, values in ( + ("Service prerequisites", self.prerequisites), + ("Service evidence types", self.required_evidence_types), + ("Service fee references", self.fee_refs), + ("Service deadline references", self.deadline_refs), + ("Service channels", self.channels), + ("Service remedy references", self.remedy_refs), + ("Service level references", self.service_level_refs), + ): + _validate_text_tuple(values, label) + _validate_reference_kind( + self.responsible_organization_ref, + "organization_unit", + "service responsible organization", + ) + _validate_reference_kind( + self.responsible_function_ref, + "function", + "service responsible function", + ) + _validate_reference_kind(self.mandate_ref, "mandate", "service mandate") + _validate_reference_kind( + self.derived_from_ref, + "service", + "parent service definition", + ) + for item in self.jurisdiction_refs: + _validate_reference_kind(item, "jurisdiction", "service jurisdiction") + requirement_keys = tuple( + item.key for item in self.availability_requirements + ) + if len(requirement_keys) != len(set(requirement_keys)): + raise InstitutionalContextError( + "Service availability requirements cannot contain duplicates." + ) + _validate_reference_tenants( + self.reference.tenant_id, + ( + self.responsible_organization_ref, + self.responsible_function_ref, + self.mandate_ref, + self.derived_from_ref, + *self.jurisdiction_refs, + ), + ) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "reference": self.reference.to_dict(), + "key": self.key, + "temporal": self.temporal.to_dict(), + "title": self.title, + "audience": list(self.audience), + "prerequisites": list(self.prerequisites), + "legal_bases": [ + item.to_dict(include_inspection=include_inspection) + for item in self.legal_bases + ], + "required_evidence_types": list(self.required_evidence_types), + "fee_refs": list(self.fee_refs), + "deadline_refs": list(self.deadline_refs), + "channels": list(self.channels), + "responsible_organization_ref": _reference_value( + self.responsible_organization_ref + ), + "responsible_function_ref": _reference_value( + self.responsible_function_ref + ), + "mandate_ref": _reference_value(self.mandate_ref), + "jurisdiction_refs": [ + item.to_dict() for item in self.jurisdiction_refs + ], + "bindings": [item.to_dict() for item in self.bindings], + "availability_requirements": [ + item.to_dict() for item in self.availability_requirements + ], + "remedy_refs": list(self.remedy_refs), + "service_level_refs": list(self.service_level_refs), + "publication_state": self.publication_state, + "availability_explanation_ref": self.availability_explanation_ref, + "derived_from_ref": _reference_value(self.derived_from_ref), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "ServiceDefinition": + return cls( + reference=InstitutionalReference.from_mapping( + _required_mapping(value, "reference") + ), + key=_required_text(value, "key"), + temporal=TemporalRevision.from_mapping( + _required_mapping(value, "temporal") + ), + title=_required_text(value, "title"), + audience=_text_tuple(value.get("audience")), + prerequisites=_text_tuple(value.get("prerequisites")), + legal_bases=tuple( + LegalBasisReference.from_mapping(item) + for item in _mapping_sequence(value.get("legal_bases")) + ), + required_evidence_types=_text_tuple( + value.get("required_evidence_types") + ), + fee_refs=_text_tuple(value.get("fee_refs")), + deadline_refs=_text_tuple(value.get("deadline_refs")), + channels=_text_tuple(value.get("channels")), + responsible_organization_ref=_optional_institutional_reference( + value.get("responsible_organization_ref") + ), + responsible_function_ref=_optional_institutional_reference( + value.get("responsible_function_ref") + ), + mandate_ref=_optional_institutional_reference( + value.get("mandate_ref") + ), + jurisdiction_refs=_institutional_references( + value.get("jurisdiction_refs") + ), + bindings=tuple( + ServiceBinding.from_mapping(item) + for item in _mapping_sequence(value.get("bindings")) + ), + availability_requirements=tuple( + ServiceAvailabilityRequirement.from_mapping(item) + for item in _mapping_sequence( + value.get("availability_requirements") + ) + ), + remedy_refs=_text_tuple(value.get("remedy_refs")), + service_level_refs=_text_tuple(value.get("service_level_refs")), + publication_state=cast( + ServicePublicationState, + str(value.get("publication_state") or "draft"), + ), + availability_explanation_ref=_optional_text( + value.get("availability_explanation_ref") + ), + derived_from_ref=_optional_institutional_reference( + value.get("derived_from_ref") + ), + ) + + +def derive_service_restriction( + parent: ServiceDefinition, + *, + reference: InstitutionalReference, + temporal: TemporalRevision, + title: str | None = None, + audience: Sequence[str] | None = None, + add_prerequisites: Sequence[str] = (), + add_required_evidence_types: Sequence[str] = (), + channels: Sequence[str] | None = None, + add_legal_bases: Sequence[LegalBasisReference] = (), + add_bindings: Sequence[ServiceBinding] = (), + publication_state: ServicePublicationState | None = None, + availability_explanation_ref: str | None = None, +) -> ServiceDefinition: + """Derive a Service specialization that cannot widen parent constraints.""" + + _validate_reference_kind(reference, "service", "derived service definition") + if reference.tenant_id != parent.reference.tenant_id: + raise InstitutionalContextError( + "A Service restriction cannot cross tenants." + ) + if reference.version != temporal.revision: + raise InstitutionalContextError( + "Derived Service reference version must match its temporal revision." + ) + if temporal.recorded_at is None or not str( + temporal.change_reason or "" + ).strip(): + raise InstitutionalContextError( + "A derived Service requires recorded_at and change_reason." + ) + if parent.temporal.valid_from is not None and ( + temporal.valid_from is None + or temporal.valid_from < parent.temporal.valid_from + ): + raise InstitutionalContextError( + "A derived Service cannot start before its parent." + ) + if parent.temporal.valid_to is not None and ( + temporal.valid_to is None or temporal.valid_to > parent.temporal.valid_to + ): + raise InstitutionalContextError( + "A derived Service cannot remain valid beyond its parent." + ) + + narrowed_audience = tuple(audience) if audience is not None else parent.audience + if not set(narrowed_audience).issubset(parent.audience): + raise InstitutionalContextError( + "A derived Service cannot widen its parent audience." + ) + narrowed_channels = ( + tuple(channels) if channels is not None else parent.channels + ) + if not set(narrowed_channels).issubset(parent.channels): + raise InstitutionalContextError( + "A derived Service cannot widen its parent channels." + ) + next_publication_state = publication_state or parent.publication_state + allowed_publication_states: Mapping[ + ServicePublicationState, + frozenset[ServicePublicationState], + ] = { + "draft": frozenset({"draft"}), + "published": frozenset({"draft", "published", "suspended", "retired"}), + "suspended": frozenset({"suspended", "retired"}), + "retired": frozenset({"retired"}), + } + if next_publication_state not in allowed_publication_states[ + parent.publication_state + ]: + raise InstitutionalContextError( + "A derived Service cannot widen its parent publication state." + ) + + return replace( + parent, + reference=reference, + temporal=temporal, + title=title if title is not None else parent.title, + audience=narrowed_audience, + prerequisites=_ordered_text_union( + parent.prerequisites, + add_prerequisites, + ), + required_evidence_types=_ordered_text_union( + parent.required_evidence_types, + add_required_evidence_types, + ), + channels=narrowed_channels, + legal_bases=tuple(dict.fromkeys((*parent.legal_bases, *add_legal_bases))), + bindings=tuple(dict.fromkeys((*parent.bindings, *add_bindings))), + publication_state=next_publication_state, + availability_explanation_ref=( + availability_explanation_ref + if availability_explanation_ref is not None + else parent.availability_explanation_ref + ), + derived_from_ref=parent.reference, + ) + + +@runtime_checkable +class ServiceDefinitionProvider(Protocol): + def get_service_definition( + self, + session: object, + principal: object, + *, + reference: InstitutionalReference, + effective_at: datetime | None = None, + ) -> ServiceDefinition | None: ... + + def list_service_definitions( + self, + session: object, + principal: object, + *, + tenant_id: str, + query: str = "", + limit: int = 100, + ) -> Sequence[ServiceDefinition]: ... + + +@dataclass(frozen=True, slots=True) +class FormFieldDefinition: + key: str + label: str + value_type: FormValueType = "text" + required: bool = False + help_text: str | None = None + options: tuple[str, ...] = () + constraints: Mapping[str, object] = field(default_factory=dict) + default_value: object | None = None + + def __post_init__(self) -> None: + _require_identifier(self.key, "Form field key") + _require_text(self.label, "Form field label") + if self.value_type not in _FORM_VALUE_TYPES: + raise InstitutionalContextError( + f"Unsupported form field value type: {self.value_type!r}." + ) + if not isinstance(self.required, bool): + raise InstitutionalContextError("Form field required must be a boolean.") + _validate_text_tuple(self.options, "Form field options") + if self.value_type in {"choice", "multi_choice"} and not self.options: + raise InstitutionalContextError( + "Choice form fields require at least one option." + ) + if self.value_type not in {"choice", "multi_choice"} and self.options: + raise InstitutionalContextError( + "Only choice form fields may declare options." + ) + if not isinstance(self.constraints, Mapping): + raise InstitutionalContextError( + "Form field constraints must be an object." + ) + + def to_dict(self) -> dict[str, object]: + return { + "key": self.key, + "label": self.label, + "value_type": self.value_type, + "required": self.required, + "help_text": self.help_text, + "options": list(self.options), + "constraints": dict(self.constraints), + "default_value": self.default_value, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "FormFieldDefinition": + constraints = value.get("constraints") + if constraints is not None and not isinstance(constraints, Mapping): + raise InstitutionalContextError( + "Form field constraints must be an object." + ) + return cls( + key=_required_text(value, "key"), + label=_required_text(value, "label"), + value_type=cast(FormValueType, str(value.get("value_type") or "text")), + required=_boolean_value(value, "required", default=False), + help_text=_optional_text(value.get("help_text")), + options=_text_tuple(value.get("options")), + constraints=dict(constraints or {}), + default_value=value.get("default_value"), + ) + + +@dataclass(frozen=True, slots=True) +class FormDefinition: + reference: InstitutionalReference + key: str + temporal: TemporalRevision + title: str + fields: tuple[FormFieldDefinition, ...] + description: str | None = None + publication_state: FormPublicationState = "draft" + allow_drafts: bool = True + max_attachments: int = 0 + signature_requirement: FormSignatureRequirement = "none" + policy_refs: tuple[str, ...] = () + handoff_kinds: tuple[str, ...] = () + metadata: Mapping[str, object] = field(default_factory=dict) + + def __post_init__(self) -> None: + _validate_reference_kind(self.reference, "form", "form definition") + _require_identifier(self.key, "Form key") + _require_text(self.title, "Form title") + if not self.reference.version: + raise InstitutionalContextError( + "A Form definition requires an exact reference version." + ) + if self.reference.version != self.temporal.revision: + raise InstitutionalContextError( + "Form reference version must match its temporal revision." + ) + if self.publication_state not in _FORM_PUBLICATION_STATES: + raise InstitutionalContextError( + f"Unsupported form publication state: {self.publication_state!r}." + ) + if not self.fields: + raise InstitutionalContextError( + "A Form definition requires at least one field." + ) + field_keys = tuple(item.key for item in self.fields) + if len(field_keys) != len(set(field_keys)): + raise InstitutionalContextError( + "Form definition field keys must be unique." + ) + if not isinstance(self.allow_drafts, bool): + raise InstitutionalContextError("Form allow_drafts must be a boolean.") + if not isinstance(self.max_attachments, int) or isinstance( + self.max_attachments, bool + ) or not 0 <= self.max_attachments <= 1000: + raise InstitutionalContextError( + "Form max_attachments must be between 0 and 1000." + ) + if self.signature_requirement not in _FORM_SIGNATURE_REQUIREMENTS: + raise InstitutionalContextError( + "Unsupported form signature requirement." + ) + _validate_text_tuple(self.policy_refs, "Form policy references") + _validate_text_tuple(self.handoff_kinds, "Form handoff kinds") + if not isinstance(self.metadata, Mapping): + raise InstitutionalContextError("Form metadata must be an object.") + + def to_dict(self) -> dict[str, object]: + return { + "reference": self.reference.to_dict(disclose_label=True), + "key": self.key, + "temporal": self.temporal.to_dict(), + "title": self.title, + "description": self.description, + "fields": [item.to_dict() for item in self.fields], + "publication_state": self.publication_state, + "allow_drafts": self.allow_drafts, + "max_attachments": self.max_attachments, + "signature_requirement": self.signature_requirement, + "policy_refs": list(self.policy_refs), + "handoff_kinds": list(self.handoff_kinds), + "metadata": dict(self.metadata), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "FormDefinition": + metadata = value.get("metadata") + if metadata is not None and not isinstance(metadata, Mapping): + raise InstitutionalContextError("Form metadata must be an object.") + raw_max_attachments = value.get("max_attachments", 0) + if not isinstance(raw_max_attachments, int) or isinstance( + raw_max_attachments, bool + ): + raise InstitutionalContextError( + "Form max_attachments must be an integer." + ) + return cls( + reference=InstitutionalReference.from_mapping( + _required_mapping(value, "reference") + ), + key=_required_text(value, "key"), + temporal=TemporalRevision.from_mapping( + _required_mapping(value, "temporal") + ), + title=_required_text(value, "title"), + description=_optional_text(value.get("description")), + fields=tuple( + FormFieldDefinition.from_mapping(item) + for item in _mapping_sequence(value.get("fields")) + ), + publication_state=cast( + FormPublicationState, + str(value.get("publication_state") or "draft"), + ), + allow_drafts=_boolean_value(value, "allow_drafts", default=True), + max_attachments=raw_max_attachments, + signature_requirement=cast( + FormSignatureRequirement, + str(value.get("signature_requirement") or "none"), + ), + policy_refs=_text_tuple(value.get("policy_refs")), + handoff_kinds=_text_tuple(value.get("handoff_kinds")), + metadata=dict(metadata or {}), + ) + + +@runtime_checkable +class FormDefinitionProvider(Protocol): + def get_form_definition( + self, + session: object, + principal: object, + *, + reference: InstitutionalReference, + effective_at: datetime | None = None, + ) -> FormDefinition | None: ... + + def list_form_definitions( + self, + session: object, + principal: object, + *, + tenant_id: str, + query: str = "", + limit: int = 100, + ) -> Sequence[FormDefinition]: ... + + +@dataclass(frozen=True, slots=True) +class ServiceLaunchRequest: + service_ref: InstitutionalReference + binding: ServiceBinding + idempotency_key: str + requested_at: datetime + parameters: Mapping[str, object] + + def __post_init__(self) -> None: + _validate_reference_kind(self.service_ref, "service", "service launch") + if not self.service_ref.version: + raise InstitutionalContextError( + "Service launch requires an exact Service revision." + ) + _require_text(self.idempotency_key, "Service launch idempotency key") + _require_aware_datetime(self.requested_at, "Service launch requested_at") + if len(self.parameters) > 100: + raise InstitutionalContextError( + "Service launch parameters are limited to 100 entries." + ) + for key in self.parameters: + _require_text(str(key), "Service launch parameter key") + + +@dataclass(frozen=True, slots=True) +class ServiceLaunchResult: + service_ref: InstitutionalReference + binding: ServiceBinding + state: ServiceLaunchState + metadata: Mapping[str, object] + target_ref: InstitutionalReference | None = None + href: str | None = None + replayed: bool = False + evidence: tuple[EvidenceReference, ...] = () + + def __post_init__(self) -> None: + _validate_reference_kind(self.service_ref, "service", "service launch result") + if not self.service_ref.version: + raise InstitutionalContextError( + "Service launch result requires an exact Service revision." + ) + if self.state not in {"started", "redirect"}: + raise InstitutionalContextError("Unsupported Service launch state.") + if self.target_ref is not None and ( + self.target_ref.tenant_id != self.service_ref.tenant_id + ): + raise InstitutionalContextError( + "Service launch result cannot target another tenant." + ) + if any( + item.tenant_id != self.service_ref.tenant_id for item in self.evidence + ): + raise InstitutionalContextError( + "Service launch evidence cannot cross tenants." + ) + _validate_service_launch_href(self.href) + if self.state == "redirect" and self.href is None: + raise InstitutionalContextError( + "Redirect Service launches require a target URL." + ) + if self.state == "started" and self.target_ref is None and self.href is None: + raise InstitutionalContextError( + "Started Service launches require a target reference or URL." + ) + + def to_dict(self) -> dict[str, object]: + return { + "service_ref": self.service_ref.to_dict(), + "binding": self.binding.to_dict(), + "state": self.state, + "target_ref": _reference_value(self.target_ref), + "href": self.href, + "replayed": self.replayed, + "evidence": [ + item.to_dict(include_inspection=False) for item in self.evidence + ], + "metadata": dict(self.metadata), + } + + +def service_launch_capability(binding_kind: str) -> str: + capabilities = { + "case": "cases.service_launcher", + "form": "forms_runtime.service_launcher", + "workflow": "workflow_engine.service_launcher", + } + try: + return capabilities[binding_kind] + except KeyError as exc: + raise InstitutionalContextError( + f"Service binding kind {binding_kind!r} has no launch capability." + ) from exc + + +@runtime_checkable +class ServiceLauncher(Protocol): + def launch_service( + self, + session: object, + principal: object, + *, + definition: ServiceDefinition, + request: ServiceLaunchRequest, + ) -> ServiceLaunchResult: ... + + +@runtime_checkable +class ServiceAvailabilityEvaluator(Protocol): + def evaluate_service_availability( + self, + session: object, + principal: object, + *, + definition: ServiceDefinition, + effective_at: datetime, + ) -> ServiceAvailabilityAssessment: ... + + +@dataclass(frozen=True, slots=True) +class PartySubjectReference: + kind: PartySubjectKind + provider: str + subject_id: str + tenant_id: str + version: str | None = None + + def __post_init__(self) -> None: + if self.kind not in _PARTY_SUBJECT_KINDS: + raise InstitutionalContextError( + f"Unsupported party subject kind: {self.kind!r}." + ) + _require_text(self.provider, "Party subject provider") + _require_identifier(self.subject_id, "Party subject id") + _require_identifier(self.tenant_id, "Party subject tenant id") + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "provider": self.provider, + "subject_id": self.subject_id, + "tenant_id": self.tenant_id, + "version": self.version, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "PartySubjectReference": + return cls( + kind=cast(PartySubjectKind, _required_text(value, "kind")), + provider=_required_text(value, "provider"), + subject_id=_required_text(value, "subject_id"), + tenant_id=_required_text(value, "tenant_id"), + version=_optional_text(value.get("version")), + ) + + +@dataclass(frozen=True, slots=True) +class PartyRepresentation: + representative_party_ref: InstitutionalReference + represented_party_ref: InstitutionalReference + power_ref: str + permitted_actions: tuple[str, ...] + temporal: TemporalRevision + evidence: tuple[EvidenceReference, ...] = () + revoked_at: datetime | None = None + + def __post_init__(self) -> None: + _validate_reference_kind( + self.representative_party_ref, "party", "representative party" + ) + _validate_reference_kind( + self.represented_party_ref, "party", "represented party" + ) + if ( + self.representative_party_ref.tenant_id + != self.represented_party_ref.tenant_id + ): + raise InstitutionalContextError( + "Party representation cannot cross tenants." + ) + _require_identifier(self.power_ref, "Party representation power ref") + _validate_text_tuple( + self.permitted_actions, + "Party representation permitted actions", + require=True, + ) + _require_aware_datetime(self.revoked_at, "party representation revoked_at") + for item in self.evidence: + if item.tenant_id != self.representative_party_ref.tenant_id: + raise InstitutionalContextError( + "Party representation evidence belongs to a different tenant." + ) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "representative_party_ref": self.representative_party_ref.to_dict(), + "represented_party_ref": self.represented_party_ref.to_dict(), + "power_ref": self.power_ref, + "permitted_actions": list(self.permitted_actions), + "temporal": self.temporal.to_dict(), + "evidence": [ + item.to_dict(include_inspection=include_inspection) + for item in self.evidence + ], + "revoked_at": _datetime_value(self.revoked_at), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "PartyRepresentation": + return cls( + representative_party_ref=InstitutionalReference.from_mapping( + _required_mapping(value, "representative_party_ref") + ), + represented_party_ref=InstitutionalReference.from_mapping( + _required_mapping(value, "represented_party_ref") + ), + power_ref=_required_text(value, "power_ref"), + permitted_actions=_text_tuple(value.get("permitted_actions")), + temporal=TemporalRevision.from_mapping( + _required_mapping(value, "temporal") + ), + evidence=tuple( + EvidenceReference.from_mapping(item) + for item in _mapping_sequence(value.get("evidence")) + ), + revoked_at=_optional_datetime(value.get("revoked_at")), + ) + + +@dataclass(frozen=True, slots=True) +class ProcedureParty: + reference: InstitutionalReference + procedure_ref: InstitutionalReference + role: str + subject: PartySubjectReference + temporal: TemporalRevision + preferred_channels: tuple[str, ...] = () + permitted_channels: tuple[str, ...] = () + delivery_recipient: bool = False + representations: tuple[PartyRepresentation, ...] = () + contact_snapshot_refs: tuple[str, ...] = () + evidence: tuple[EvidenceReference, ...] = () + status: ProcedurePartyStatus = "active" + + def __post_init__(self) -> None: + _validate_reference_kind(self.reference, "party", "procedure party") + if self.procedure_ref.kind not in {"case", "workflow", "decision"}: + raise InstitutionalContextError( + "Procedure party must belong to a case, workflow, or decision." + ) + if self.subject.tenant_id != self.reference.tenant_id: + raise InstitutionalContextError( + "Party subject belongs to a different tenant." + ) + _validate_reference_tenants( + self.reference.tenant_id, (self.procedure_ref,) + ) + _require_text(self.role, "Procedure party role") + if self.status not in _PROCEDURE_PARTY_STATUSES: + raise InstitutionalContextError( + f"Unsupported procedure party status: {self.status!r}." + ) + _validate_text_tuple( + self.preferred_channels, + "Procedure party preferred channels", + ) + _validate_text_tuple( + self.permitted_channels, + "Procedure party permitted channels", + ) + _validate_text_tuple( + self.contact_snapshot_refs, + "Procedure party contact snapshots", + ) + if set(self.preferred_channels) - set(self.permitted_channels): + raise InstitutionalContextError( + "Preferred party channels must also be permitted." + ) + for representation in self.representations: + if ( + representation.representative_party_ref.tenant_id + != self.reference.tenant_id + ): + raise InstitutionalContextError( + "Party representation belongs to a different tenant." + ) + for item in self.evidence: + if item.tenant_id != self.reference.tenant_id: + raise InstitutionalContextError( + "Procedure party evidence belongs to a different tenant." + ) + + def to_dict(self, *, include_inspection: bool = False) -> dict[str, object]: + return { + "reference": self.reference.to_dict(), + "procedure_ref": self.procedure_ref.to_dict(), + "role": self.role, + "subject": self.subject.to_dict(), + "temporal": self.temporal.to_dict(), + "preferred_channels": list(self.preferred_channels), + "permitted_channels": list(self.permitted_channels), + "delivery_recipient": self.delivery_recipient, + "representations": [ + item.to_dict(include_inspection=include_inspection) + for item in self.representations + ], + "contact_snapshot_refs": list(self.contact_snapshot_refs), + "evidence": [ + item.to_dict(include_inspection=include_inspection) + for item in self.evidence + ], + "status": self.status, + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "ProcedureParty": + return cls( + reference=InstitutionalReference.from_mapping( + _required_mapping(value, "reference") + ), + procedure_ref=InstitutionalReference.from_mapping( + _required_mapping(value, "procedure_ref") + ), + role=_required_text(value, "role"), + subject=PartySubjectReference.from_mapping( + _required_mapping(value, "subject") + ), + temporal=TemporalRevision.from_mapping( + _required_mapping(value, "temporal") + ), + preferred_channels=_text_tuple(value.get("preferred_channels")), + permitted_channels=_text_tuple(value.get("permitted_channels")), + delivery_recipient=_boolean_value( + value, + "delivery_recipient", + default=False, + ), + representations=tuple( + PartyRepresentation.from_mapping(item) + for item in _mapping_sequence(value.get("representations")) + ), + contact_snapshot_refs=_text_tuple( + value.get("contact_snapshot_refs") + ), + evidence=tuple( + EvidenceReference.from_mapping(item) + for item in _mapping_sequence(value.get("evidence")) + ), + status=cast( + ProcedurePartyStatus, + str(value.get("status") or "active"), + ), + ) + + +def revise_procedure_party( + current: ProcedureParty, + *, + expected_revision: str, + temporal: TemporalRevision, + status: ProcedurePartyStatus = "active", + role: str | None = None, + subject: PartySubjectReference | None = None, + preferred_channels: Sequence[str] | None = None, + permitted_channels: Sequence[str] | None = None, + delivery_recipient: bool | None = None, + representations: Sequence[PartyRepresentation] | None = None, + contact_snapshot_refs: Sequence[str] | None = None, + evidence: Sequence[EvidenceReference] | None = None, +) -> ProcedureParty: + """Correct or close a procedure-party assignment as a new revision.""" + + if current.temporal.revision != expected_revision: + raise InstitutionalContextError( + "Procedure party revision conflict: the expected revision is stale." + ) + if temporal.revision == current.temporal.revision: + raise InstitutionalContextError( + "A procedure party revision requires a new revision identifier." + ) + if temporal.recorded_at is None or not str( + temporal.change_reason or "" + ).strip(): + raise InstitutionalContextError( + "A procedure party revision requires recorded_at and change_reason." + ) + if status not in _PROCEDURE_PARTY_TRANSITIONS[current.status]: + raise InstitutionalContextError( + f"Procedure party transition {current.status!r} to {status!r} is not allowed." + ) + return replace( + current, + reference=replace( + current.reference, + version=temporal.revision, + valid_at=temporal.valid_from or current.reference.valid_at, + ), + temporal=temporal, + status=status, + role=role if role is not None else current.role, + subject=subject if subject is not None else current.subject, + preferred_channels=( + tuple(preferred_channels) + if preferred_channels is not None + else current.preferred_channels + ), + permitted_channels=( + tuple(permitted_channels) + if permitted_channels is not None + else current.permitted_channels + ), + delivery_recipient=( + delivery_recipient + if delivery_recipient is not None + else current.delivery_recipient + ), + representations=( + tuple(representations) + if representations is not None + else current.representations + ), + contact_snapshot_refs=( + tuple(contact_snapshot_refs) + if contact_snapshot_refs is not None + else current.contact_snapshot_refs + ), + evidence=tuple(evidence) if evidence is not None else current.evidence, + ) + + +def revoke_party_representation( + current: PartyRepresentation, + *, + expected_revision: str, + temporal: TemporalRevision, + revoked_at: datetime, + evidence: Sequence[EvidenceReference] | None = None, +) -> PartyRepresentation: + """Record effective revocation of representation authority with OCC.""" + + if current.temporal.revision != expected_revision: + raise InstitutionalContextError( + "Party representation revision conflict: the expected revision is stale." + ) + if current.revoked_at is not None: + raise InstitutionalContextError("Party representation is already revoked.") + if temporal.revision == current.temporal.revision: + raise InstitutionalContextError( + "A party representation revision requires a new revision identifier." + ) + if temporal.recorded_at is None or not str( + temporal.change_reason or "" + ).strip(): + raise InstitutionalContextError( + "A party representation revision requires recorded_at and change_reason." + ) + _require_aware_datetime(revoked_at, "party representation revoked_at") + if temporal.valid_from is not None and revoked_at < temporal.valid_from: + raise InstitutionalContextError( + "Party representation revocation cannot precede its valid interval." + ) + return replace( + current, + temporal=temporal, + revoked_at=revoked_at, + evidence=tuple(evidence) if evidence is not None else current.evidence, + ) + + +@runtime_checkable +class PartyResolver(Protocol): + def list_procedure_parties( + self, + session: object, + principal: object, + *, + procedure_ref: InstitutionalReference, + effective_at: datetime | None = None, + ) -> Sequence[ProcedureParty]: ... + + +@dataclass(frozen=True, slots=True) +class DecisionEffectReference: + effect_key: str + state: str + resource_refs: tuple[str, ...] = () + audit_event_refs: tuple[str, ...] = () + evidence_refs: tuple[str, ...] = () + + def __post_init__(self) -> None: + _require_text(self.effect_key, "Decision effect key") + _require_text(self.state, "Decision effect state") + for label, values in ( + ("Decision effect resource references", self.resource_refs), + ("Decision effect audit references", self.audit_event_refs), + ("Decision effect evidence references", self.evidence_refs), + ): + _validate_text_tuple(values, label) + + def to_dict(self) -> dict[str, object]: + return _decision_effect_value(self) + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "DecisionEffectReference": + return cls( + effect_key=_required_text(value, "effect_key"), + state=_required_text(value, "state"), + resource_refs=_text_tuple(value.get("resource_refs")), + audit_event_refs=_text_tuple(value.get("audit_event_refs")), + evidence_refs=_text_tuple(value.get("evidence_refs")), + ) + + +@dataclass(frozen=True, slots=True) +class FormalDecision: + reference: InstitutionalReference + temporal: TemporalRevision + decision_type: str + subject_refs: tuple[InstitutionalReference, ...] + state: DecisionState + authority_context: GovernedContextEnvelope + fact_evidence: tuple[EvidenceReference, ...] + legal_bases: tuple[LegalBasisReference, ...] + assurance_level: DecisionAssuranceLevel = "human" + automation_preparation_refs: tuple[str, ...] = () + operative_result: str | None = None + reasoning: str | None = None + conditions: tuple[str, ...] = () + requested_effects: tuple[DecisionEffectReference, ...] = () + observed_effects: tuple[DecisionEffectReference, ...] = () + delivery_refs: tuple[str, ...] = () + publication_refs: tuple[str, ...] = () + remedy_refs: tuple[str, ...] = () + review_refs: tuple[str, ...] = () + correction_of_ref: InstitutionalReference | None = None + revocation_of_ref: InstitutionalReference | None = None + supersedes_ref: InstitutionalReference | None = None + + def __post_init__(self) -> None: + _validate_reference_kind(self.reference, "decision", "formal decision") + _require_text(self.decision_type, "Decision type") + if self.state not in _DECISION_STATES: + raise InstitutionalContextError( + f"Unsupported decision state: {self.state!r}." + ) + if self.assurance_level not in _DECISION_ASSURANCE_LEVELS: + raise InstitutionalContextError( + f"Unsupported decision assurance level: {self.assurance_level!r}." + ) + actor = self.authority_context.actor + if actor is None: + raise InstitutionalContextError( + "Formal decisions require a responsible actor." + ) + if ( + self.assurance_level == "automated_under_mandate" + and actor.service_account_id is None + ): + raise InstitutionalContextError( + "Automated formal decisions require a responsible service account." + ) + if ( + self.assurance_level != "automated_under_mandate" + and actor.account_id is None + ): + raise InstitutionalContextError( + "Human-assured formal decisions require a responsible account." + ) + if not self.subject_refs: + raise InstitutionalContextError( + "Formal decisions require at least one subject reference." + ) + if self.authority_context.tenant_id != self.reference.tenant_id: + raise InstitutionalContextError( + "Decision authority context belongs to a different tenant." + ) + if self.authority_context.mandate_ref is None: + raise InstitutionalContextError( + "Formal decisions require a mandate in their authority context." + ) + if not self.fact_evidence or not self.legal_bases: + raise InstitutionalContextError( + "Formal decisions require fact evidence and legal bases." + ) + if self.state in {"decided", "effective", "corrected"} and not str( + self.operative_result or "" + ).strip(): + raise InstitutionalContextError( + "A decided formal outcome requires an operative result." + ) + if ( + self.authority_context.decision_ref is not None + and not _same_reference_identity( + self.authority_context.decision_ref, + self.reference, + ) + ): + raise InstitutionalContextError( + "Decision authority context references a different decision." + ) + _validate_reference_tenants( + self.reference.tenant_id, + ( + *self.subject_refs, + self.correction_of_ref, + self.revocation_of_ref, + self.supersedes_ref, + ), + ) + for item in ( + self.correction_of_ref, + self.revocation_of_ref, + self.supersedes_ref, + ): + _validate_reference_kind(item, "decision", "decision history reference") + for item in self.fact_evidence: + if item.tenant_id != self.reference.tenant_id: + raise InstitutionalContextError( + "Decision evidence belongs to a different tenant." + ) + for label, values in ( + ( + "Decision automation preparation references", + self.automation_preparation_refs, + ), + ("Decision conditions", self.conditions), + ("Decision delivery references", self.delivery_refs), + ("Decision publication references", self.publication_refs), + ("Decision remedy references", self.remedy_refs), + ("Decision review references", self.review_refs), + ): + _validate_text_tuple(values, label) + + def to_dict(self, *, include_protected: bool = False) -> dict[str, object]: + return { + "reference": self.reference.to_dict(disclose_label=include_protected), + "temporal": self.temporal.to_dict(), + "decision_type": self.decision_type, + "subject_refs": [item.to_dict() for item in self.subject_refs], + "state": self.state, + "authority_context": self.authority_context.to_dict( + include_inspection=include_protected + ), + "fact_evidence": [ + item.to_dict(include_inspection=include_protected) + for item in self.fact_evidence + ], + "legal_bases": [ + item.to_dict(include_inspection=include_protected) + for item in self.legal_bases + ], + "assurance_level": self.assurance_level, + "automation_preparation_refs": list( + self.automation_preparation_refs + ), + "operative_result": self.operative_result if include_protected else None, + "reasoning": self.reasoning if include_protected else None, + "conditions": list(self.conditions) if include_protected else [], + "requested_effects": [ + _decision_effect_value(item) for item in self.requested_effects + ], + "observed_effects": [ + _decision_effect_value(item) for item in self.observed_effects + ], + "delivery_refs": list(self.delivery_refs), + "publication_refs": list(self.publication_refs), + "remedy_refs": list(self.remedy_refs), + "review_refs": list(self.review_refs), + "correction_of_ref": _reference_value(self.correction_of_ref), + "revocation_of_ref": _reference_value(self.revocation_of_ref), + "supersedes_ref": _reference_value(self.supersedes_ref), + } + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "FormalDecision": + return cls( + reference=InstitutionalReference.from_mapping( + _required_mapping(value, "reference") + ), + temporal=TemporalRevision.from_mapping( + _required_mapping(value, "temporal") + ), + decision_type=_required_text(value, "decision_type"), + subject_refs=_institutional_references(value.get("subject_refs")), + state=cast(DecisionState, _required_text(value, "state")), + authority_context=GovernedContextEnvelope.from_mapping( + _required_mapping(value, "authority_context") + ), + fact_evidence=tuple( + EvidenceReference.from_mapping(item) + for item in _mapping_sequence(value.get("fact_evidence")) + ), + legal_bases=tuple( + LegalBasisReference.from_mapping(item) + for item in _mapping_sequence(value.get("legal_bases")) + ), + assurance_level=cast( + DecisionAssuranceLevel, + str(value.get("assurance_level") or "human"), + ), + automation_preparation_refs=_text_tuple( + value.get("automation_preparation_refs") + ), + operative_result=_optional_text(value.get("operative_result")), + reasoning=_optional_text(value.get("reasoning")), + conditions=_text_tuple(value.get("conditions")), + requested_effects=tuple( + DecisionEffectReference.from_mapping(item) + for item in _mapping_sequence(value.get("requested_effects")) + ), + observed_effects=tuple( + DecisionEffectReference.from_mapping(item) + for item in _mapping_sequence(value.get("observed_effects")) + ), + delivery_refs=_text_tuple(value.get("delivery_refs")), + publication_refs=_text_tuple(value.get("publication_refs")), + remedy_refs=_text_tuple(value.get("remedy_refs")), + review_refs=_text_tuple(value.get("review_refs")), + correction_of_ref=_optional_institutional_reference( + value.get("correction_of_ref") + ), + revocation_of_ref=_optional_institutional_reference( + value.get("revocation_of_ref") + ), + supersedes_ref=_optional_institutional_reference( + value.get("supersedes_ref") + ), + ) + + +def revise_formal_decision( + current: FormalDecision, + *, + expected_revision: str, + temporal: TemporalRevision, + state: DecisionState, + operative_result: str | None = None, + reasoning: str | None = None, + conditions: Sequence[str] | None = None, + requested_effects: Sequence[DecisionEffectReference] | None = None, + observed_effects: Sequence[DecisionEffectReference] | None = None, + delivery_refs: Sequence[str] | None = None, + publication_refs: Sequence[str] | None = None, + remedy_refs: Sequence[str] | None = None, + review_refs: Sequence[str] | None = None, + assurance_level: DecisionAssuranceLevel | None = None, + automation_preparation_refs: Sequence[str] | None = None, +) -> FormalDecision: + """Create an immutable, OCC-guarded revision of a formal decision.""" + + if current.temporal.revision != expected_revision: + raise InstitutionalContextError( + "Formal decision revision conflict: the expected revision is stale." + ) + if temporal.revision == current.temporal.revision: + raise InstitutionalContextError( + "A formal decision revision requires a new revision identifier." + ) + if temporal.recorded_at is None or not str( + temporal.change_reason or "" + ).strip(): + raise InstitutionalContextError( + "A formal decision revision requires recorded_at and change_reason." + ) + if state not in _DECISION_TRANSITIONS[current.state]: + raise InstitutionalContextError( + f"Formal decision transition {current.state!r} to {state!r} is not allowed." + ) + + previous_reference = current.reference + revised_reference = replace( + current.reference, + version=temporal.revision, + valid_at=temporal.valid_from or current.reference.valid_at, + ) + authority_context = replace( + current.authority_context, + temporal=temporal, + decision_ref=revised_reference, + ) + return FormalDecision( + reference=revised_reference, + temporal=temporal, + decision_type=current.decision_type, + subject_refs=current.subject_refs, + state=state, + authority_context=authority_context, + fact_evidence=current.fact_evidence, + legal_bases=current.legal_bases, + assurance_level=( + assurance_level + if assurance_level is not None + else current.assurance_level + ), + automation_preparation_refs=( + tuple(automation_preparation_refs) + if automation_preparation_refs is not None + else current.automation_preparation_refs + ), + operative_result=( + operative_result + if operative_result is not None + else current.operative_result + ), + reasoning=reasoning if reasoning is not None else current.reasoning, + conditions=( + tuple(conditions) if conditions is not None else current.conditions + ), + requested_effects=( + tuple(requested_effects) + if requested_effects is not None + else current.requested_effects + ), + observed_effects=( + tuple(observed_effects) + if observed_effects is not None + else current.observed_effects + ), + delivery_refs=( + tuple(delivery_refs) + if delivery_refs is not None + else current.delivery_refs + ), + publication_refs=( + tuple(publication_refs) + if publication_refs is not None + else current.publication_refs + ), + remedy_refs=( + tuple(remedy_refs) + if remedy_refs is not None + else current.remedy_refs + ), + review_refs=( + tuple(review_refs) + if review_refs is not None + else current.review_refs + ), + correction_of_ref=( + previous_reference + if state == "corrected" + else current.correction_of_ref + ), + revocation_of_ref=( + previous_reference + if state == "revoked" + else current.revocation_of_ref + ), + supersedes_ref=previous_reference, + ) + + +@runtime_checkable +class DecisionRegistry(Protocol): + def get_decision( + self, + session: object, + principal: object, + *, + reference: InstitutionalReference, + ) -> FormalDecision | None: ... + + def record_decision( + self, + session: object, + principal: object, + *, + decision: FormalDecision, + expected_revision: str | None = None, + ) -> FormalDecision: ... + + +def _decision_effect_value(item: DecisionEffectReference) -> dict[str, object]: + return { + "effect_key": item.effect_key, + "state": item.state, + "resource_refs": list(item.resource_refs), + "audit_event_refs": list(item.audit_event_refs), + "evidence_refs": list(item.evidence_refs), + } + + +def _validate_reference_kind( + reference: InstitutionalReference | None, + expected: InstitutionalReferenceKind, + label: str, +) -> None: + if reference is not None and reference.kind != expected: + raise InstitutionalContextError( + f"{label.capitalize()} must have kind {expected!r}, not {reference.kind!r}." + ) + + +def _validate_reference_tenants( + tenant_id: str, + references: Sequence[InstitutionalReference | None], +) -> None: + for reference in references: + if reference is not None and reference.tenant_id != tenant_id: + raise InstitutionalContextError( + f"Institutional reference {reference.kind}:{reference.object_id} " + "belongs to a different tenant." + ) + + +def _reference_value( + reference: InstitutionalReference | None, +) -> dict[str, object] | None: + return reference.to_dict() if reference is not None else None + + +def _same_reference_identity( + left: InstitutionalReference, + right: InstitutionalReference, +) -> bool: + return ( + left.kind, + left.owner_module, + left.object_id, + left.tenant_id, + left.version, + left.valid_at, + ) == ( + right.kind, + right.owner_module, + right.object_id, + right.tenant_id, + right.version, + right.valid_at, + ) + + +def _stable_reference_key( + reference: InstitutionalReference, +) -> tuple[str, str, str, str]: + return ( + reference.kind, + reference.owner_module, + reference.object_id, + reference.tenant_id, + ) + + +def _reference_is_allowed( + reference: InstitutionalReference | None, + allowed: Sequence[InstitutionalReference], +) -> bool: + if reference is None: + return False + key = _stable_reference_key(reference) + return any(_stable_reference_key(item) == key for item in allowed) + + +def _ordered_text_union( + current: Sequence[str], + additions: Sequence[str], +) -> tuple[str, ...]: + return tuple(dict.fromkeys((*current, *additions))) + + +def _require_identifier(value: str, label: str) -> None: + candidate = str(value or "").strip() + if not _REFERENCE_ID_RE.fullmatch(candidate): + raise InstitutionalContextError(f"{label} is invalid.") + + +def _require_text(value: str, label: str) -> None: + if not str(value or "").strip(): + raise InstitutionalContextError(f"{label} is required.") + + +def _require_aware_datetime(value: datetime | None, label: str) -> None: + if value is not None and (value.tzinfo is None or value.utcoffset() is None): + raise InstitutionalContextError(f"{label} must include a timezone.") + + +def _datetime_value(value: datetime | None) -> str | None: + return value.isoformat() if value is not None else None + + +def _optional_datetime(value: object | None) -> datetime | None: + if value is None or value == "": + return None + if isinstance(value, datetime): + return value + try: + return datetime.fromisoformat(str(value)) + except ValueError as exc: + raise InstitutionalContextError( + f"Invalid ISO datetime value: {value!r}." + ) from exc + + +def _required_text(value: Mapping[str, object], key: str) -> str: + result = _optional_text(value.get(key)) + if result is None: + raise InstitutionalContextError(f"Institutional payload {key} is required.") + return result + + +def _required_mapping( + value: Mapping[str, object], + key: str, +) -> Mapping[str, object]: + result = value.get(key) + if not isinstance(result, Mapping): + raise InstitutionalContextError( + f"Institutional payload {key} must be an object." + ) + return result + + +def _required_datetime(value: object | None) -> datetime: + result = _optional_datetime(value) + if result is None: + raise InstitutionalContextError( + "Institutional payload datetime is required." + ) + return result + + +def _boolean_value( + value: Mapping[str, object], + key: str, + *, + default: bool, +) -> bool: + result = value.get(key, default) + if not isinstance(result, bool): + raise InstitutionalContextError( + f"Institutional payload {key} must be a boolean." + ) + return result + + +def _optional_text(value: object | None) -> str | None: + if value is None: + return None + result = str(value).strip() + return result or None + + +def _text_tuple(value: object | None) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str): + return (value,) + if not isinstance(value, Sequence): + raise InstitutionalContextError("Expected a list of text values.") + return tuple(str(item) for item in value) + + +def _validate_text_tuple( + value: tuple[str, ...], + label: str, + *, + require: bool = False, +) -> None: + if require and not value: + raise InstitutionalContextError(f"{label} are required.") + if any(not item.strip() for item in value): + raise InstitutionalContextError(f"{label} cannot contain empty values.") + if len(value) != len(set(value)): + raise InstitutionalContextError(f"{label} cannot contain duplicates.") + + +def _optional_float(value: object | None) -> float | None: + return None if value is None else float(value) + + +def _validate_inspection_url(value: str | None) -> None: + if value is None: + return + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise InstitutionalContextError( + "Inspection URLs must use HTTP or HTTPS." + ) + if parsed.username is not None or parsed.password is not None: + raise InstitutionalContextError( + "Inspection URLs must not contain credentials." + ) + + +def _validate_service_launch_href(value: str | None) -> None: + if value is None: + return + if value.startswith("/") and not value.startswith("//"): + return + parsed = urlsplit(value) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise InstitutionalContextError( + "Service launch URLs must be relative application paths or credential-free HTTP(S) URLs." + ) + + +def _mapping_sequence(value: object | None) -> tuple[Mapping[str, object], ...]: + if value is None: + return () + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise InstitutionalContextError("Expected a list of object payloads.") + if any(not isinstance(item, Mapping) for item in value): + raise InstitutionalContextError("Expected object payloads in list.") + return cast(tuple[Mapping[str, object], ...], tuple(value)) + + +def _optional_institutional_reference( + value: object | None, +) -> InstitutionalReference | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise InstitutionalContextError( + "Institutional references must be object payloads." + ) + return InstitutionalReference.from_mapping(value) + + +def _institutional_references( + value: object | None, +) -> tuple[InstitutionalReference, ...]: + return tuple( + InstitutionalReference.from_mapping(item) for item in _mapping_sequence(value) + ) + + +def _optional_actor(value: object | None) -> ActorRepresentationReference | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise InstitutionalContextError("Actor context must be an object payload.") + return ActorRepresentationReference.from_mapping(value) + + +def _optional_governance( + value: object | None, +) -> InformationGovernanceReference | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise InstitutionalContextError( + "Information governance must be an object payload." + ) + return InformationGovernanceReference.from_mapping(value) + + +def _optional_presentation(value: object | None) -> PresentationReference | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise InstitutionalContextError( + "Presentation context must be an object payload." + ) + return PresentationReference.from_mapping(value) + + +def _external_reference_from_mapping( + value: Mapping[str, object], +) -> ExternalObjectReference: + return ExternalObjectReference( + system=_required_text(value, "system"), + object_type=_required_text(value, "object_type"), + object_id=_required_text(value, "object_id"), + maturity=cast( + IntegrationMaturity, str(value.get("maturity") or "link") + ), + authority_mode=cast( + SourceAuthorityMode, + str(value.get("authority_mode") or "linked_reference"), + ), + connector_id=_optional_text(value.get("connector_id")), + canonical_url=_optional_text(value.get("canonical_url")), + version=_optional_text(value.get("version")), + etag=_optional_text(value.get("etag")), + observed_at=_optional_datetime(value.get("observed_at")), + metadata=( + dict(value.get("metadata")) + if isinstance(value.get("metadata"), Mapping) + else {} + ), + ) + + +__all__ = [ + "CAPABILITY_DECISION_REGISTRY", + "CAPABILITY_FORM_DEFINITIONS", + "CAPABILITY_MANDATE_RESOLVER", + "CAPABILITY_PARTY_RESOLVER", + "CAPABILITY_SERVICE_DEFINITIONS", + "CAPABILITY_SERVICE_AVAILABILITY", + "INSTITUTIONAL_CONTEXT_CONTRACT_VERSION", + "ActorRepresentationReference", + "DecisionEffectReference", + "DecisionAssuranceLevel", + "DecisionRegistry", + "EvidenceReference", + "ExternalSourceReference", + "FormDefinition", + "FormDefinitionProvider", + "FormFieldDefinition", + "FormPublicationState", + "FormSignatureRequirement", + "FormValueType", + "FormalDecision", + "GeoReference", + "GovernedContextEnvelope", + "InformationGovernanceReference", + "InstitutionalContextError", + "InstitutionalReference", + "LegalBasisReference", + "MandateDefinition", + "MandateResolution", + "MandateResolutionRequest", + "MandateResolver", + "MandateStatus", + "PartyRepresentation", + "PartyResolver", + "PartySubjectReference", + "PresentationReference", + "ProcedureParty", + "ProcedurePartyStatus", + "ServiceBinding", + "ServiceAvailabilityAssessment", + "ServiceAvailabilityEvaluator", + "ServiceAvailabilityRequirement", + "ServiceDefinition", + "ServiceDefinitionProvider", + "ServiceLaunchRequest", + "ServiceLaunchResult", + "ServiceLauncher", + "ServiceLaunchState", + "ServicePublicationState", + "TemporalRevision", + "derive_service_restriction", + "service_launch_capability", + "resolve_mandate_candidates", + "revise_procedure_party", + "revise_mandate_definition", + "revise_formal_decision", + "revoke_party_representation", +] diff --git a/src/govoplan_core/core/module_installer.py b/src/govoplan_core/core/module_installer.py index 8b60c21..da2c2d5 100644 --- a/src/govoplan_core/core/module_installer.py +++ b/src/govoplan_core/core/module_installer.py @@ -335,6 +335,18 @@ def module_install_preflight( issues.append(ModuleInstallerIssue("warning", "empty_plan", "No planned package changes are present.")) if not maintenance_mode: issues.append(ModuleInstallerIssue("blocker", "maintenance_required", "Package changes require maintenance mode.")) + if os.getenv("GOVOPLAN_STATE_PROFILE", "local").strip().lower() == "shared": + issues.append( + ModuleInstallerIssue( + "blocker", + "immutable_cluster_release_required", + ( + "Shared-state deployments cannot mutate packages on one runtime node. " + "Build and roll out one immutable release image across every API, worker, " + "and scheduler node." + ), + ) + ) activation_candidates = desired_modules_after_package_plan(desired_sequence, plan) issues.extend(module_manifest_compatibility_issues(available, module_ids=activation_candidates)) diff --git a/src/govoplan_core/core/module_package_catalog.py b/src/govoplan_core/core/module_package_catalog.py index 637b022..68b5fbf 100644 --- a/src/govoplan_core/core/module_package_catalog.py +++ b/src/govoplan_core/core/module_package_catalog.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 import binascii from collections import defaultdict +from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -16,6 +17,11 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range +from govoplan_core.core.provider_governance import ( + external_provider_from_mapping, + module_architecture_from_mapping, + module_architecture_issues, +) from govoplan_core.security.http_fetch import fetch_http_text, is_http_url _INTERFACE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$") @@ -613,6 +619,72 @@ def _normalize_catalog_item(value: Any) -> dict[str, object]: "notes": _optional_str(value, "notes"), "tags": _string_list(value.get("tags")), } + raw_architecture = value.get("architecture") + if raw_architecture is not None: + if not isinstance(raw_architecture, Mapping): + raise ValueError( + f"Module package catalog architecture for {module_id!r} must be an object." + ) + architecture = module_architecture_from_mapping(raw_architecture) + issues = module_architecture_issues( + architecture, + has_migrations=bool(item["migration_tasks"]) + or bool(item["migration_notes"]), + ) + if issues: + raise ValueError( + f"Module package catalog architecture for {module_id!r} is invalid: " + + "; ".join(issues) + ) + item["architecture"] = architecture.to_dict() + raw_providers = value.get("external_providers") + if raw_providers is not None: + if not isinstance(raw_providers, list): + raise ValueError( + f"Module package catalog external_providers for {module_id!r} must be a list." + ) + providers = [] + seen_provider_ids: set[str] = set() + for raw_provider in raw_providers: + if not isinstance(raw_provider, Mapping): + raise ValueError( + f"Module package catalog external provider entries for {module_id!r} must be objects." + ) + provider = external_provider_from_mapping(raw_provider) + if provider.module_id != module_id: + raise ValueError( + f"Module package catalog provider {provider.id!r} belongs to " + f"{provider.module_id!r}, not {module_id!r}." + ) + if provider.id in seen_provider_ids: + raise ValueError( + f"Module package catalog has duplicate provider {provider.id!r}." + ) + seen_provider_ids.add(provider.id) + providers.append(provider) + if providers and "architecture" not in item: + raise ValueError( + f"Module package catalog {module_id!r} declares external providers without architecture metadata." + ) + if providers: + architecture_payload = item["architecture"] + if not isinstance(architecture_payload, Mapping): + raise ValueError( + f"Module package catalog {module_id!r} has invalid architecture metadata." + ) + architecture_modes = set( + _string_list(architecture_payload.get("supported_authority_modes")) + ) + provider_modes = { + mode for provider in providers for mode in provider.authority_modes + } + missing_modes = provider_modes - architecture_modes + if missing_modes: + raise ValueError( + f"Module package catalog {module_id!r} provider modes are missing from architecture metadata: " + + ", ".join(sorted(missing_modes)) + ) + item["external_providers"] = [provider.to_dict() for provider in providers] if not version_range_is_valid( version_min=item["current_version_min"] if isinstance(item["current_version_min"], str) else None, version_max_exclusive=item["current_version_max_exclusive"] if isinstance(item["current_version_max_exclusive"], str) else None, diff --git a/src/govoplan_core/core/modules.py b/src/govoplan_core/core/modules.py index f6c04ef..899ab98 100644 --- a/src/govoplan_core/core/modules.py +++ b/src/govoplan_core/core/modules.py @@ -5,6 +5,11 @@ from dataclasses import dataclass, field from typing import Any, Literal, Protocol, TYPE_CHECKING from govoplan_core.core.ownership import OwnershipProviderRegistration +from govoplan_core.core.provider_governance import ( + ExternalProviderDeclaration, + ExternalProviderStateProviderRegistration, + ModuleArchitectureDeclaration, +) from govoplan_core.core.views import ViewSurface if TYPE_CHECKING: @@ -438,6 +443,12 @@ class ModuleManifest: "OperationalCheckProviderRegistration", ..., ] = () + architecture: ModuleArchitectureDeclaration | None = None + external_providers: tuple[ExternalProviderDeclaration, ...] = () + external_provider_state_providers: tuple[ + ExternalProviderStateProviderRegistration, + ..., + ] = () compatibility: ModuleCompatibility = field(default_factory=ModuleCompatibility) on_activate: LifecycleHook | None = None on_deactivate: LifecycleHook | None = None diff --git a/src/govoplan_core/core/object_storage.py b/src/govoplan_core/core/object_storage.py new file mode 100644 index 0000000..241651b --- /dev/null +++ b/src/govoplan_core/core/object_storage.py @@ -0,0 +1,597 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from heapq import nsmallest +import os +from pathlib import Path +import tempfile +from typing import Any, Iterable, Protocol +from urllib.parse import urlsplit + +from govoplan_core.security.outbound_http import ( + OutboundHttpError, + response_limit, + validate_unpinned_sdk_http_url, +) + + +class StorageBackendError(RuntimeError): + """Base error for the deployment-owned object-storage boundary.""" + + +class StorageObjectMissing(StorageBackendError): + """Raised when a referenced object no longer exists.""" + + +@dataclass(frozen=True, slots=True) +class StorageObjectInfo: + key: str + size_bytes: int + + +@dataclass(frozen=True, slots=True) +class StorageObjectPage: + objects: tuple[StorageObjectInfo, ...] + next_cursor: str | None = None + + +class StorageBackend(Protocol): + """Shared byte-object storage used by modules without cross-module imports.""" + + name: str + + def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str | None = None, + ) -> None: ... + + def get_bytes(self, key: str) -> bytes: ... + + def iter_bytes( + self, + key: str, + *, + chunk_size: int = 1024 * 1024, + ) -> Iterable[bytes]: ... + + def delete(self, key: str) -> None: ... + + def exists(self, key: str) -> bool: ... + + def stat(self, key: str) -> StorageObjectInfo: ... + + def list_objects( + self, + *, + prefix: str, + after: str | None = None, + limit: int = 500, + ) -> StorageObjectPage: ... + + +@dataclass(slots=True) +class LocalFilesystemStorageBackend: + root: Path + fallback_roots: tuple[Path, ...] = field(default_factory=tuple) + name: str = "local" + + def __post_init__(self) -> None: + self.root = self.root.expanduser().resolve() + self.fallback_roots = tuple( + root.expanduser().resolve() for root in self.fallback_roots if root + ) + self.root.mkdir(parents=True, exist_ok=True) + + def _path_for_root(self, root: Path, key: str) -> Path: + normalized = normalize_storage_key(key) + path = (root / normalized).resolve() + if not path.is_relative_to(root): + raise StorageBackendError("Storage key escapes local storage root") + return path + + def _path(self, key: str) -> Path: + return self._path_for_root(self.root, key) + + def _readable_path(self, key: str) -> Path: + primary = self._path(key) + if primary.exists() and primary.is_file(): + return primary + for root in self.fallback_roots: + candidate = self._path_for_root(root, key) + if candidate.exists() and candidate.is_file(): + return candidate + raise StorageObjectMissing("Stored object does not exist") + + def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str | None = None, + ) -> None: + del content_type + path = self._path(key) + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + def get_bytes(self, key: str) -> bytes: + return self._readable_path(key).read_bytes() + + def iter_bytes( + self, + key: str, + *, + chunk_size: int = 1024 * 1024, + ) -> Iterable[bytes]: + path = self._readable_path(key) + with path.open("rb") as handle: + while True: + chunk = handle.read(chunk_size) + if not chunk: + break + yield chunk + + def delete(self, key: str) -> None: + path = self._path(key) + if path.exists() and path.is_file(): + path.unlink() + + def exists(self, key: str) -> bool: + try: + self._readable_path(key) + except StorageObjectMissing: + return False + return True + + def stat(self, key: str) -> StorageObjectInfo: + path = self._readable_path(key) + return StorageObjectInfo( + key=normalize_storage_key(key), + size_bytes=path.stat().st_size, + ) + + def list_objects( + self, + *, + prefix: str, + after: str | None = None, + limit: int = 500, + ) -> StorageObjectPage: + normalized_prefix = normalize_storage_prefix(prefix) + normalized_after = normalize_storage_key(after) if after else None + normalized_limit = max(1, min(int(limit), 5000)) + + def matching_objects() -> Iterable[StorageObjectInfo]: + for path in _iter_local_files(self.root): + key = path.relative_to(self.root).as_posix() + if not key.startswith(normalized_prefix) or ( + normalized_after is not None and key <= normalized_after + ): + continue + yield StorageObjectInfo( + key=key, + size_bytes=path.stat().st_size, + ) + + candidates = nsmallest( + normalized_limit + 1, + matching_objects(), + key=lambda item: item.key, + ) + has_more = len(candidates) > normalized_limit + page = tuple(candidates[:normalized_limit]) + return StorageObjectPage( + objects=page, + next_cursor=page[-1].key if has_more and page else None, + ) + + +@dataclass(slots=True) +class S3StorageBackend: + bucket: str + endpoint_url: str + region_name: str + access_key_id: str + secret_access_key: str + deployment_managed: bool = False + endpoint_trusted: bool = False + name: str = "s3" + _client: Any = field(default=None, init=False, repr=False) + + @property + def client(self): + if self._client is not None: + return self._client + if self.deployment_managed: + endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url) + elif self.endpoint_trusted: + endpoint_url = _trusted_deployment_endpoint(self.endpoint_url) + else: + try: + endpoint_url = validate_unpinned_sdk_http_url( + self.endpoint_url, + label="Object storage S3 endpoint", + ) + except OutboundHttpError as exc: + raise StorageBackendError(str(exc)) from exc + try: + import boto3 + from botocore.config import Config + except ModuleNotFoundError as exc: + raise StorageBackendError( + "boto3 is required for the S3 storage backend" + ) from exc + options: dict[str, object] = { + "endpoint_url": endpoint_url, + "region_name": self.region_name, + "aws_access_key_id": self.access_key_id, + "aws_secret_access_key": self.secret_access_key, + } + if self.deployment_managed: + options["config"] = Config(s3={"addressing_style": "path"}) + self._client = boto3.client("s3", **options) + return self._client + + def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str | None = None, + ) -> None: + normalized = normalize_storage_key(key) + max_bytes = response_limit("file") + if len(data) > max_bytes: + raise StorageBackendError( + f"Stored object exceeds the deployment limit of {max_bytes} bytes" + ) + kwargs: dict[str, object] = { + "Bucket": self.bucket, + "Key": normalized, + "Body": data, + } + if content_type: + kwargs["ContentType"] = content_type + try: + self.client.put_object(**kwargs) + except Exception as exc: # pragma: no cover - depends on S3 backend + raise StorageBackendError(str(exc)) from exc + + def get_bytes(self, key: str) -> bytes: + normalized = normalize_storage_key(key) + try: + obj = self.client.get_object( + Bucket=self.bucket, + Key=normalized, + ) + max_bytes = response_limit("file") + body = obj["Body"] + try: + _reject_declared_object_size(obj, max_bytes=max_bytes) + data = body.read(max_bytes + 1) + if len(data) > max_bytes: + raise StorageBackendError( + "Stored object exceeds the deployment limit of " + f"{max_bytes} bytes" + ) + return data + finally: + if hasattr(body, "close"): + body.close() + except StorageBackendError: + raise + except Exception as exc: # pragma: no cover - depends on S3 backend + if _s3_missing_error(exc): + raise StorageObjectMissing("Stored object does not exist") from exc + raise StorageBackendError(str(exc)) from exc + + def iter_bytes( + self, + key: str, + *, + chunk_size: int = 1024 * 1024, + ) -> Iterable[bytes]: + normalized = normalize_storage_key(key) + try: + obj = self.client.get_object( + Bucket=self.bucket, + Key=normalized, + ) + max_bytes = response_limit("file") + body = obj["Body"] + try: + _reject_declared_object_size(obj, max_bytes=max_bytes) + total = 0 + while True: + chunk = body.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise StorageBackendError( + "Stored object exceeds the deployment limit of " + f"{max_bytes} bytes" + ) + yield chunk + finally: + if hasattr(body, "close"): + body.close() + except StorageBackendError: + raise + except Exception as exc: # pragma: no cover - depends on S3 backend + if _s3_missing_error(exc): + raise StorageObjectMissing("Stored object does not exist") from exc + raise StorageBackendError(str(exc)) from exc + + def delete(self, key: str) -> None: + try: + self.client.delete_object( + Bucket=self.bucket, + Key=normalize_storage_key(key), + ) + except Exception as exc: # pragma: no cover - depends on S3 backend + raise StorageBackendError(str(exc)) from exc + + def exists(self, key: str) -> bool: + try: + self.client.head_object( + Bucket=self.bucket, + Key=normalize_storage_key(key), + ) + return True + except Exception as exc: + if _s3_missing_error(exc): + return False + raise StorageBackendError(str(exc)) from exc + + def stat(self, key: str) -> StorageObjectInfo: + normalized = normalize_storage_key(key) + try: + response = self.client.head_object( + Bucket=self.bucket, + Key=normalized, + ) + except Exception as exc: # pragma: no cover - depends on S3 backend + if _s3_missing_error(exc): + raise StorageObjectMissing("Stored object does not exist") from exc + raise StorageBackendError(str(exc)) from exc + try: + size = int(response.get("ContentLength")) + except (AttributeError, TypeError, ValueError) as exc: + raise StorageBackendError( + "S3 object metadata did not include a valid size" + ) from exc + return StorageObjectInfo(key=normalized, size_bytes=size) + + def list_objects( + self, + *, + prefix: str, + after: str | None = None, + limit: int = 500, + ) -> StorageObjectPage: + normalized_prefix = normalize_storage_prefix(prefix) + normalized_limit = max(1, min(int(limit), 1000)) + kwargs: dict[str, object] = { + "Bucket": self.bucket, + "Prefix": normalized_prefix, + "MaxKeys": normalized_limit, + } + if after: + kwargs["StartAfter"] = normalize_storage_key(after) + try: + response = self.client.list_objects_v2(**kwargs) + except Exception as exc: # pragma: no cover - depends on S3 backend + raise StorageBackendError(str(exc)) from exc + objects = tuple( + StorageObjectInfo( + key=str(item["Key"]), + size_bytes=int(item.get("Size") or 0), + ) + for item in response.get("Contents", ()) + if isinstance(item, dict) and item.get("Key") + ) + has_more = bool(response.get("IsTruncated")) + return StorageObjectPage( + objects=objects, + next_cursor=objects[-1].key if has_more and objects else None, + ) + + +def configured_storage_backend(settings: object) -> StorageBackend: + """Build the deployment-wide object store from Core settings. + + Modules own their metadata and key namespaces. The deployment owns the + storage endpoint and credentials, so modules do not need to depend on the + Files package merely to persist opaque generated bytes. + """ + + configured = ( + str(getattr(settings, "file_storage_backend", "local") or "local") + .strip() + .lower() + ) + if configured in {"local", "filesystem", "fs"}: + raw_fallbacks = str( + getattr(settings, "file_storage_local_fallback_roots", "") or "" + ) + return LocalFilesystemStorageBackend( + Path( + str( + getattr( + settings, + "file_storage_local_root", + "runtime/files", + ) + ) + ), + fallback_roots=tuple( + Path(item.strip()) for item in raw_fallbacks.split(",") if item.strip() + ), + ) + if configured in {"s3", "garage"}: + return S3StorageBackend( + bucket=str( + getattr(settings, "file_storage_s3_bucket", None) + or getattr(settings, "s3_bucket", "files") + ), + endpoint_url=str( + getattr(settings, "file_storage_s3_endpoint_url", None) + or getattr(settings, "s3_endpoint_url", "") + ), + region_name=str( + getattr(settings, "file_storage_s3_region", None) + or getattr(settings, "s3_region", "") + ), + access_key_id=str( + getattr(settings, "file_storage_s3_access_key_id", None) + or getattr(settings, "s3_access_key_id", "") + ), + secret_access_key=str( + getattr( + settings, + "file_storage_s3_secret_access_key", + None, + ) + or getattr(settings, "s3_secret_access_key", "") + ), + deployment_managed=bool( + getattr( + settings, + "file_storage_s3_deployment_managed", + False, + ) + ), + endpoint_trusted=bool( + getattr( + settings, + "file_storage_s3_endpoint_trusted", + False, + ) + ), + ) + raise StorageBackendError(f"Unsupported object storage backend: {configured}") + + +def normalize_storage_key(value: str) -> str: + candidate = str(value or "").strip().replace("\\", "/") + parts = candidate.split("/") + if ( + not candidate + or candidate.startswith("/") + or any(part in {"", ".", ".."} for part in parts) + or any(ord(character) < 32 for character in candidate) + ): + raise StorageBackendError("Storage key is not a safe relative key") + return "/".join(parts) + + +def normalize_storage_prefix(value: str) -> str: + candidate = str(value or "").strip().replace("\\", "/") + if not candidate: + return "" + trailing_slash = candidate.endswith("/") + normalized = normalize_storage_key(candidate.rstrip("/")) + return normalized + ("/" if trailing_slash else "") + + +def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None: + if not isinstance(obj, dict): + return + try: + declared_size = int(obj.get("ContentLength")) + except (TypeError, ValueError): + return + if declared_size > max_bytes: + raise StorageBackendError( + f"Stored object exceeds the deployment limit of {max_bytes} bytes" + ) + + +def _iter_local_files(root: Path): + for entry in sorted(root.iterdir(), key=lambda item: item.name): + if entry.is_symlink(): + continue + if entry.is_dir(): + yield from _iter_local_files(entry) + elif entry.is_file(): + yield entry + + +def _s3_missing_error(exc: Exception) -> bool: + response = getattr(exc, "response", None) + if not isinstance(response, dict): + return False + error = response.get("Error") + metadata = response.get("ResponseMetadata") + code = str(error.get("Code") if isinstance(error, dict) else "") + status_code = metadata.get("HTTPStatusCode") if isinstance(metadata, dict) else None + return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404 + + +def _deployment_managed_garage_endpoint(value: str) -> str: + endpoint = str(value or "").strip() + if endpoint != "http://garage:3900": + raise StorageBackendError( + "Deployment-managed S3 trust is restricted to http://garage:3900" + ) + return endpoint + + +def _trusted_deployment_endpoint(value: str) -> str: + endpoint = str(value or "").strip() + parsed = urlsplit(endpoint) + try: + parsed.port + except ValueError as exc: + raise StorageBackendError( + "Deployment-trusted S3 endpoint has an invalid port" + ) from exc + if ( + parsed.scheme.lower() != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or parsed.path not in {"", "/"} + ): + raise StorageBackendError( + "Deployment-trusted S3 endpoint must be an HTTPS origin without " + "credentials, query, fragment, or path" + ) + return endpoint.rstrip("/") + + +__all__ = [ + "LocalFilesystemStorageBackend", + "S3StorageBackend", + "StorageBackend", + "StorageBackendError", + "StorageObjectInfo", + "StorageObjectMissing", + "StorageObjectPage", + "configured_storage_backend", + "normalize_storage_key", + "normalize_storage_prefix", +] diff --git a/src/govoplan_core/core/provider_governance.py b/src/govoplan_core/core/provider_governance.py new file mode 100644 index 0000000..ceec889 --- /dev/null +++ b/src/govoplan_core/core/provider_governance.py @@ -0,0 +1,1376 @@ +from __future__ import annotations + +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Literal, cast + +from govoplan_core.core.external_references import ( + INTEGRATION_MATURITY_ORDER, + SOURCE_AUTHORITY_MODES, + IntegrationMaturity, + SourceAuthorityMode, + integration_maturity_rank, +) + + +ArchitectureLayer = Literal[ + "runtime_meta", + "institutional_foundation", + "governance_accountability", + "human_work_procedure", + "communication_participation", + "content_records_evidence", + "data_reporting_integration", + "domain_capability", + "product_sector_package", +] +ModuleKind = Literal[ + "kernel", + "foundation", + "governance", + "runtime", + "editor", + "domain", + "integration", + "presentation", + "operations", + "meta", +] +ModuleMaturity = Literal[ + "concept", + "scaffold", + "vertical_slice", + "reference_ready", + "supported", + "lts", +] +MaturityEvidenceKind = Literal[ + "test", + "documentation", + "reference_process", + "target_test", + "migration", + "upgrade", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "compatibility", + "provider", +] +ProviderOperation = Literal[ + "discover", + "link", + "search", + "read", + "write", + "delete", + "publish", + "synchronize", + "migrate", + "preview", + "dry_run", +] +ProviderHealthState = Literal[ + "healthy", + "warning", + "error", + "unknown", + "inactive", +] +ProviderFreshnessState = Literal[ + "current", + "stale", + "unknown", + "not_applicable", +] +ProviderConflictState = Literal[ + "clear", + "pending", + "blocked", + "unknown", + "not_applicable", +] +ProviderRecoveryState = Literal[ + "ready", + "attention", + "unsupported", + "unknown", + "not_applicable", +] + + +ARCHITECTURE_LAYERS: tuple[ArchitectureLayer, ...] = ( + "runtime_meta", + "institutional_foundation", + "governance_accountability", + "human_work_procedure", + "communication_participation", + "content_records_evidence", + "data_reporting_integration", + "domain_capability", + "product_sector_package", +) +MODULE_KINDS: tuple[ModuleKind, ...] = ( + "kernel", + "foundation", + "governance", + "runtime", + "editor", + "domain", + "integration", + "presentation", + "operations", + "meta", +) +MODULE_MATURITY_ORDER: tuple[ModuleMaturity, ...] = ( + "concept", + "scaffold", + "vertical_slice", + "reference_ready", + "supported", + "lts", +) +MATURITY_EVIDENCE_KINDS: tuple[MaturityEvidenceKind, ...] = ( + "test", + "documentation", + "reference_process", + "target_test", + "migration", + "upgrade", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "compatibility", + "provider", +) +PROVIDER_OPERATIONS: tuple[ProviderOperation, ...] = ( + "discover", + "link", + "search", + "read", + "write", + "delete", + "publish", + "synchronize", + "migrate", + "preview", + "dry_run", +) +PROVIDER_HEALTH_STATES: tuple[ProviderHealthState, ...] = ( + "healthy", + "warning", + "error", + "unknown", + "inactive", +) +PROVIDER_FRESHNESS_STATES: tuple[ProviderFreshnessState, ...] = ( + "current", + "stale", + "unknown", + "not_applicable", +) +PROVIDER_CONFLICT_STATES: tuple[ProviderConflictState, ...] = ( + "clear", + "pending", + "blocked", + "unknown", + "not_applicable", +) +PROVIDER_RECOVERY_STATES: tuple[ProviderRecoveryState, ...] = ( + "ready", + "attention", + "unsupported", + "unknown", + "not_applicable", +) + +_MATURITY_EVIDENCE_REQUIREMENTS: dict[ModuleMaturity, frozenset[str]] = { + "concept": frozenset(), + "scaffold": frozenset({"documentation"}), + "vertical_slice": frozenset({"test", "documentation"}), + "reference_ready": frozenset( + { + "test", + "documentation", + "reference_process", + "target_test", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + } + ), + "supported": frozenset( + { + "test", + "documentation", + "reference_process", + "target_test", + "security", + "operations", + "upgrade", + "recovery", + "accessibility", + "privacy", + } + ), + "lts": frozenset( + { + "test", + "documentation", + "reference_process", + "target_test", + "security", + "operations", + "upgrade", + "recovery", + "accessibility", + "privacy", + "compatibility", + } + ), +} +_OPERATION_MINIMUM_MATURITY: dict[ProviderOperation, IntegrationMaturity] = { + "discover": "discover", + "link": "link", + "search": "search", + "read": "read", + "write": "publish", + "delete": "publish", + "publish": "publish", + "synchronize": "synchronize", + "migrate": "migrate", + "preview": "read", + "dry_run": "read", +} +_EFFECT_OPERATIONS = frozenset({"write", "delete", "publish", "synchronize", "migrate"}) +_READ_OPERATIONS = frozenset({"search", "read", "preview", "dry_run"}) + + +class ArchitectureDeclarationError(ValueError): + pass + + +@dataclass(frozen=True, slots=True) +class ModuleMaturityEvidence: + kind: MaturityEvidenceKind + reference: str + summary: str + + def __post_init__(self) -> None: + if self.kind not in MATURITY_EVIDENCE_KINDS: + raise ArchitectureDeclarationError( + f"Unsupported module maturity evidence kind: {self.kind!r}." + ) + _require_text(self.reference, "Maturity evidence reference") + _require_text(self.summary, "Maturity evidence summary") + + def to_dict(self) -> dict[str, str]: + return { + "kind": self.kind, + "reference": self.reference, + "summary": self.summary, + } + + +@dataclass(frozen=True, slots=True) +class ModuleArchitectureDocumentation: + migration: tuple[str, ...] = () + upgrade: tuple[str, ...] = () + recovery: tuple[str, ...] = () + security: tuple[str, ...] = () + operations: tuple[str, ...] = () + + def __post_init__(self) -> None: + for field_name in ("migration", "upgrade", "recovery", "security", "operations"): + _validate_text_tuple( + getattr(self, field_name), + f"Architecture documentation {field_name}", + ) + + def to_dict(self) -> dict[str, list[str]]: + return { + "migration": list(self.migration), + "upgrade": list(self.upgrade), + "recovery": list(self.recovery), + "security": list(self.security), + "operations": list(self.operations), + } + + +@dataclass(frozen=True, slots=True) +class ModuleArchitectureDeclaration: + layer: ArchitectureLayer + kind: ModuleKind + maturity: ModuleMaturity + evidence: tuple[ModuleMaturityEvidence, ...] = () + known_limits: tuple[str, ...] = () + supported_authority_modes: tuple[SourceAuthorityMode, ...] = () + owned_concepts: tuple[str, ...] = () + non_owned_concepts: tuple[str, ...] = () + reference_packages: tuple[str, ...] = () + target_tested_providers: tuple[str, ...] = () + documentation: ModuleArchitectureDocumentation = field( + default_factory=ModuleArchitectureDocumentation + ) + contract_version: str = "1" + + def __post_init__(self) -> None: + if self.contract_version != "1": + raise ArchitectureDeclarationError( + "Unsupported module architecture contract version." + ) + if self.layer not in ARCHITECTURE_LAYERS: + raise ArchitectureDeclarationError( + f"Unsupported architecture layer: {self.layer!r}." + ) + if self.kind not in MODULE_KINDS: + raise ArchitectureDeclarationError( + f"Unsupported module kind: {self.kind!r}." + ) + if self.maturity not in MODULE_MATURITY_ORDER: + raise ArchitectureDeclarationError( + f"Unsupported module maturity: {self.maturity!r}." + ) + for mode in self.supported_authority_modes: + if mode not in SOURCE_AUTHORITY_MODES: + raise ArchitectureDeclarationError( + f"Unsupported source-authority mode: {mode!r}." + ) + for field_name in ( + "known_limits", + "owned_concepts", + "non_owned_concepts", + "reference_packages", + "target_tested_providers", + ): + _validate_text_tuple(getattr(self, field_name), field_name.replace("_", " ")) + if set(self.owned_concepts) & set(self.non_owned_concepts): + raise ArchitectureDeclarationError( + "Owned and explicitly non-owned concepts must not overlap." + ) + evidence_keys = [(item.kind, item.reference) for item in self.evidence] + if len(evidence_keys) != len(set(evidence_keys)): + raise ArchitectureDeclarationError( + "Module maturity evidence entries must be unique." + ) + + def to_dict(self) -> dict[str, object]: + return { + "contract_version": self.contract_version, + "layer": self.layer, + "kind": self.kind, + "maturity": self.maturity, + "evidence": [item.to_dict() for item in self.evidence], + "known_limits": list(self.known_limits), + "supported_authority_modes": list(self.supported_authority_modes), + "owned_concepts": list(self.owned_concepts), + "non_owned_concepts": list(self.non_owned_concepts), + "reference_packages": list(self.reference_packages), + "target_tested_providers": list(self.target_tested_providers), + "documentation": self.documentation.to_dict(), + } + + +def declared_module_architecture( + *, + layer: ArchitectureLayer, + kind: ModuleKind, + maturity: ModuleMaturity, + documentation_ref: str, + known_limits: Sequence[str], + test_ref: str | None = None, + supported_authority_modes: Sequence[SourceAuthorityMode] = (), + owned_concepts: Sequence[str] = (), + non_owned_concepts: Sequence[str] = (), + reference_packages: Sequence[str] = (), + target_tested_providers: Sequence[str] = (), + migration_docs: Sequence[str] = (), + upgrade_docs: Sequence[str] = (), + recovery_docs: Sequence[str] = (), + security_docs: Sequence[str] = (), + operations_docs: Sequence[str] = (), +) -> ModuleArchitectureDeclaration: + """Build concise, explicit repository-owned architecture metadata. + + The helper deliberately does not infer maturity from repository contents. + Callers must choose the claim and provide the evidence paths that support + it; workspace checks subsequently prove that repository-local paths exist. + """ + + evidence = [ + ModuleMaturityEvidence( + kind="documentation", + reference=documentation_ref, + summary="Defines the module boundary, supported behavior, and known limits.", + ) + ] + if test_ref is not None: + evidence.append( + ModuleMaturityEvidence( + kind="test", + reference=test_ref, + summary="Exercises the declared module contract and implemented slice.", + ) + ) + return ModuleArchitectureDeclaration( + layer=layer, + kind=kind, + maturity=maturity, + evidence=tuple(evidence), + known_limits=tuple(known_limits), + supported_authority_modes=tuple(supported_authority_modes), + owned_concepts=tuple(owned_concepts), + non_owned_concepts=tuple(non_owned_concepts), + reference_packages=tuple(reference_packages), + target_tested_providers=tuple(target_tested_providers), + documentation=ModuleArchitectureDocumentation( + migration=tuple(migration_docs), + upgrade=tuple(upgrade_docs), + recovery=tuple(recovery_docs), + security=tuple(security_docs), + operations=tuple(operations_docs), + ), + ) + + +@dataclass(frozen=True, slots=True) +class ProviderObjectDeclaration: + object_type: str + field_groups: tuple[str, ...] + authority_modes: tuple[SourceAuthorityMode, ...] + default_authority_mode: SourceAuthorityMode + + def __post_init__(self) -> None: + _require_text(self.object_type, "Provider object type") + _validate_text_tuple(self.field_groups, "Provider field groups", require=True) + if not self.authority_modes: + raise ArchitectureDeclarationError( + "Provider object declarations require source-authority modes." + ) + for mode in self.authority_modes: + if mode not in SOURCE_AUTHORITY_MODES: + raise ArchitectureDeclarationError( + f"Unsupported source-authority mode: {mode!r}." + ) + if self.default_authority_mode not in self.authority_modes: + raise ArchitectureDeclarationError( + "Provider default authority mode must be one of its supported modes." + ) + + def to_dict(self) -> dict[str, object]: + return { + "object_type": self.object_type, + "field_groups": list(self.field_groups), + "authority_modes": list(self.authority_modes), + "default_authority_mode": self.default_authority_mode, + } + + +@dataclass(frozen=True, slots=True) +class ProviderBehaviorDeclaration: + revision_tokens: str | None = None + concurrency: str | None = None + freshness: str | None = None + health: str | None = None + max_read_items: int | None = None + idempotency: str | None = None + retry: str | None = None + timeout_seconds: int | None = None + conflicts: str | None = None + outcome_unknown: str | None = None + outcome_unknown_supported: bool = False + evidence: str | None = None + audit_event_types: tuple[str, ...] = () + correction: str | None = None + rollback: str | None = None + compensation: str | None = None + reconciliation: str | None = None + outage: str | None = None + classifications: tuple[str, ...] = () + purposes: tuple[str, ...] = () + retention: str | None = None + secret_handling: str | None = None + + def __post_init__(self) -> None: + if self.max_read_items is not None and self.max_read_items < 1: + raise ArchitectureDeclarationError( + "Provider max_read_items must be positive." + ) + if self.timeout_seconds is not None and self.timeout_seconds < 1: + raise ArchitectureDeclarationError( + "Provider timeout_seconds must be positive." + ) + for field_name in ("audit_event_types", "classifications", "purposes"): + _validate_text_tuple(getattr(self, field_name), field_name.replace("_", " ")) + + def to_dict(self) -> dict[str, object]: + return { + "revision_tokens": self.revision_tokens, + "concurrency": self.concurrency, + "freshness": self.freshness, + "health": self.health, + "max_read_items": self.max_read_items, + "idempotency": self.idempotency, + "retry": self.retry, + "timeout_seconds": self.timeout_seconds, + "conflicts": self.conflicts, + "outcome_unknown": self.outcome_unknown, + "outcome_unknown_supported": self.outcome_unknown_supported, + "evidence": self.evidence, + "audit_event_types": list(self.audit_event_types), + "correction": self.correction, + "rollback": self.rollback, + "compensation": self.compensation, + "reconciliation": self.reconciliation, + "outage": self.outage, + "classifications": list(self.classifications), + "purposes": list(self.purposes), + "retention": self.retention, + "secret_handling": self.secret_handling, + } + + +@dataclass(frozen=True, slots=True) +class ExternalProviderDeclaration: + id: str + module_id: str + label: str + maturity: IntegrationMaturity + operations: tuple[ProviderOperation, ...] + objects: tuple[ProviderObjectDeclaration, ...] + behavior: ProviderBehaviorDeclaration + capability_names: tuple[str, ...] = () + interface_names: tuple[str, ...] = () + action_keys: tuple[str, ...] = () + effect_keys: tuple[str, ...] = () + operational_check_ids: tuple[str, ...] = () + documentation_topic_ids: tuple[str, ...] = () + policy_hooks: tuple[str, ...] = () + ownership_resource_types: tuple[str, ...] = () + contract_version: str = "1" + + def __post_init__(self) -> None: + if self.contract_version != "1": + raise ArchitectureDeclarationError( + "Unsupported external provider declaration contract version." + ) + _require_text(self.id, "Provider declaration id") + _require_text(self.module_id, "Provider declaration module id") + _require_text(self.label, "Provider declaration label") + if not self.id.startswith(f"{self.module_id}."): + raise ArchitectureDeclarationError( + "Provider declaration ids must be namespaced by their module id." + ) + if self.maturity not in INTEGRATION_MATURITY_ORDER: + raise ArchitectureDeclarationError( + f"Unsupported integration maturity: {self.maturity!r}." + ) + if not self.operations: + raise ArchitectureDeclarationError( + "Provider declarations require at least one operation." + ) + for operation in self.operations: + if operation not in PROVIDER_OPERATIONS: + raise ArchitectureDeclarationError( + f"Unsupported provider operation: {operation!r}." + ) + minimum = _OPERATION_MINIMUM_MATURITY[operation] + if integration_maturity_rank(self.maturity) < integration_maturity_rank( + minimum + ): + raise ArchitectureDeclarationError( + f"Provider operation {operation!r} requires integration maturity " + f"{minimum!r} or higher." + ) + if len(self.operations) != len(set(self.operations)): + raise ArchitectureDeclarationError( + "Provider declaration operations must be unique." + ) + if not self.objects: + raise ArchitectureDeclarationError( + "Provider declarations require at least one object declaration." + ) + object_types = [item.object_type for item in self.objects] + if len(object_types) != len(set(object_types)): + raise ArchitectureDeclarationError( + "Provider object types must be unique within a declaration." + ) + for field_name in ( + "capability_names", + "interface_names", + "action_keys", + "effect_keys", + "operational_check_ids", + "documentation_topic_ids", + "policy_hooks", + "ownership_resource_types", + ): + _validate_text_tuple(getattr(self, field_name), field_name.replace("_", " ")) + _validate_provider_authority_modes(self) + _validate_provider_behavior(self) + + @property + def authority_modes(self) -> tuple[SourceAuthorityMode, ...]: + return tuple( + dict.fromkeys( + mode for item in self.objects for mode in item.authority_modes + ) + ) + + def to_dict(self) -> dict[str, object]: + return { + "contract_version": self.contract_version, + "id": self.id, + "module_id": self.module_id, + "label": self.label, + "maturity": self.maturity, + "operations": list(self.operations), + "objects": [item.to_dict() for item in self.objects], + "authority_modes": list(self.authority_modes), + "behavior": self.behavior.to_dict(), + "capability_names": list(self.capability_names), + "interface_names": list(self.interface_names), + "action_keys": list(self.action_keys), + "effect_keys": list(self.effect_keys), + "operational_check_ids": list(self.operational_check_ids), + "documentation_topic_ids": list(self.documentation_topic_ids), + "policy_hooks": list(self.policy_hooks), + "ownership_resource_types": list(self.ownership_resource_types), + } + + +@dataclass(frozen=True, slots=True) +class ExternalProviderStateContext: + """Bounded context supplied to a module-owned runtime state provider.""" + + session: object | None + tenant_id: str | None = None + principal: object | None = None + max_items: int = 500 + + def __post_init__(self) -> None: + if self.tenant_id is not None: + _require_text(self.tenant_id, "Provider state tenant id") + if not 1 <= self.max_items <= 1_000: + raise ArchitectureDeclarationError( + "Provider state max_items must be between 1 and 1000." + ) + + +@dataclass(frozen=True, slots=True) +class ExternalProviderRuntimeState: + """One configured external binding observation without secret material.""" + + provider_id: str + observed_at: datetime + configured: bool + active: bool + health: ProviderHealthState + freshness: ProviderFreshnessState + conflict: ProviderConflictState + recovery: ProviderRecoveryState + binding_ref: str | None = None + authority_mode: SourceAuthorityMode | None = None + last_success_at: datetime | None = None + detail: str | None = None + metrics: Mapping[str, str | int | float | bool | None] = field( + default_factory=dict + ) + contract_version: str = "1" + + def __post_init__(self) -> None: + if self.contract_version != "1": + raise ArchitectureDeclarationError( + "Unsupported external provider runtime-state contract version." + ) + _require_text(self.provider_id, "Provider runtime-state id") + _validate_aware_datetime(self.observed_at, "Provider state observed_at") + if self.active and not self.configured: + raise ArchitectureDeclarationError( + "An active provider binding must be configured." + ) + if self.configured: + _require_text(self.binding_ref or "", "Configured provider binding ref") + if self.authority_mode not in SOURCE_AUTHORITY_MODES: + raise ArchitectureDeclarationError( + "Configured provider bindings require a source-authority mode." + ) + elif self.binding_ref is not None or self.authority_mode is not None: + raise ArchitectureDeclarationError( + "Unconfigured provider state cannot identify a binding or authority mode." + ) + if self.health not in PROVIDER_HEALTH_STATES: + raise ArchitectureDeclarationError( + f"Unsupported provider health state: {self.health!r}." + ) + if self.freshness not in PROVIDER_FRESHNESS_STATES: + raise ArchitectureDeclarationError( + f"Unsupported provider freshness state: {self.freshness!r}." + ) + if self.conflict not in PROVIDER_CONFLICT_STATES: + raise ArchitectureDeclarationError( + f"Unsupported provider conflict state: {self.conflict!r}." + ) + if self.recovery not in PROVIDER_RECOVERY_STATES: + raise ArchitectureDeclarationError( + f"Unsupported provider recovery state: {self.recovery!r}." + ) + if self.last_success_at is not None: + _validate_aware_datetime( + self.last_success_at, + "Provider state last_success_at", + ) + if self.detail is not None and len(self.detail.strip()) > 1_000: + raise ArchitectureDeclarationError( + "Provider state detail is limited to 1000 characters." + ) + if len(self.metrics) > 50: + raise ArchitectureDeclarationError( + "Provider state metrics are limited to 50 values." + ) + for key, value in self.metrics.items(): + _require_text(str(key), "Provider state metric name") + if not isinstance(value, (str, int, float, bool, type(None))): + raise ArchitectureDeclarationError( + "Provider state metrics must contain scalar values." + ) + + def to_dict(self) -> dict[str, object]: + return { + "contract_version": self.contract_version, + "provider_id": self.provider_id, + "binding_ref": self.binding_ref, + "authority_mode": self.authority_mode, + "configured": self.configured, + "active": self.active, + "health": self.health, + "freshness": self.freshness, + "conflict": self.conflict, + "recovery": self.recovery, + "observed_at": self.observed_at.isoformat(), + "last_success_at": ( + self.last_success_at.isoformat() + if self.last_success_at is not None + else None + ), + "detail": self.detail, + "metrics": dict(self.metrics), + } + + +ExternalProviderStateProvider = Callable[ + [ExternalProviderStateContext], + Sequence[ExternalProviderRuntimeState], +] + + +@dataclass(frozen=True, slots=True) +class ExternalProviderStateProviderRegistration: + module_id: str + provider_id: str + provider: ExternalProviderStateProvider + + def __post_init__(self) -> None: + _require_text(self.module_id, "Provider-state module id") + _require_text(self.provider_id, "Provider-state provider id") + + +def collect_external_provider_states( + registrations: Sequence[ExternalProviderStateProviderRegistration], + context: ExternalProviderStateContext, +) -> dict[str, dict[str, object]]: + """Collect and aggregate module state while isolating optional failures.""" + + result: dict[str, dict[str, object]] = {} + for registration in registrations: + if registration.provider_id in result: + raise ArchitectureDeclarationError( + f"Duplicate runtime state provider {registration.provider_id!r}." + ) + try: + states = tuple(registration.provider(context)) + if len(states) > context.max_items: + states = states[: context.max_items] + truncated = True + else: + truncated = False + if any(item.provider_id != registration.provider_id for item in states): + raise ArchitectureDeclarationError( + "Runtime state provider returned a state for a different provider." + ) + binding_refs = [ + item.binding_ref for item in states if item.binding_ref is not None + ] + if len(binding_refs) != len(set(binding_refs)): + raise ArchitectureDeclarationError( + "Runtime state provider returned duplicate binding references." + ) + if not states: + states = ( + _unconfigured_provider_state(registration.provider_id), + ) + result[registration.provider_id] = _aggregate_provider_states( + states, + truncated=truncated, + ) + except Exception as exc: # noqa: BLE001 - isolate optional provider failures. + failure = ExternalProviderRuntimeState( + provider_id=registration.provider_id, + observed_at=_runtime_now(), + configured=False, + active=False, + health="error", + freshness="unknown", + conflict="unknown", + recovery="attention", + detail=( + "Runtime state collection failed " + f"({type(exc).__name__})." + ), + ) + result[registration.provider_id] = _aggregate_provider_states( + (failure,), + truncated=False, + ) + return result + + +def module_architecture_issues( + declaration: ModuleArchitectureDeclaration, + *, + has_migrations: bool, +) -> tuple[str, ...]: + issues: list[str] = [] + evidence_kinds = {item.kind for item in declaration.evidence} + required = set(_MATURITY_EVIDENCE_REQUIREMENTS[declaration.maturity]) + if declaration.maturity in {"supported", "lts"} and has_migrations: + required.add("migration") + if ( + declaration.maturity in {"reference_ready", "supported", "lts"} + and declaration.target_tested_providers + ): + required.add("provider") + missing = sorted(required - evidence_kinds) + if missing: + issues.append( + f"maturity {declaration.maturity!r} is missing evidence: " + + ", ".join(missing) + ) + if declaration.maturity in { + "concept", + "scaffold", + "vertical_slice", + "reference_ready", + } and not declaration.known_limits: + issues.append( + f"maturity {declaration.maturity!r} must state at least one known limit" + ) + if ( + declaration.maturity in {"reference_ready", "supported", "lts"} + and not declaration.reference_packages + ): + issues.append( + f"{declaration.maturity} maturity requires a reference package" + ) + documentation_requirements: dict[ModuleMaturity, tuple[str, ...]] = { + "concept": (), + "scaffold": (), + "vertical_slice": (), + "reference_ready": ("recovery", "security", "operations"), + "supported": ("upgrade", "recovery", "security", "operations"), + "lts": ("upgrade", "recovery", "security", "operations"), + } + missing_documentation = tuple( + name + for name in documentation_requirements[declaration.maturity] + if not getattr(declaration.documentation, name) + ) + if has_migrations and declaration.maturity in {"supported", "lts"}: + if not declaration.documentation.migration: + missing_documentation = (*missing_documentation, "migration") + if missing_documentation: + issues.append( + f"maturity {declaration.maturity!r} is missing documentation: " + + ", ".join(sorted(set(missing_documentation))) + ) + return tuple(issues) + + +def module_architecture_from_mapping( + value: Mapping[str, object], +) -> ModuleArchitectureDeclaration: + raw_documentation = value.get("documentation") + documentation = ( + raw_documentation if isinstance(raw_documentation, Mapping) else {} + ) + return ModuleArchitectureDeclaration( + contract_version=str(value.get("contract_version") or "1"), + layer=cast(ArchitectureLayer, _mapping_text(value, "layer")), + kind=cast(ModuleKind, _mapping_text(value, "kind")), + maturity=cast(ModuleMaturity, _mapping_text(value, "maturity")), + evidence=tuple( + ModuleMaturityEvidence( + kind=cast( + MaturityEvidenceKind, _mapping_text(item, "kind") + ), + reference=_mapping_text(item, "reference"), + summary=_mapping_text(item, "summary"), + ) + for item in _mapping_items(value.get("evidence"), "architecture evidence") + ), + known_limits=_mapping_texts(value.get("known_limits"), "known limits"), + supported_authority_modes=tuple( + cast(SourceAuthorityMode, item) + for item in _mapping_texts( + value.get("supported_authority_modes"), + "supported authority modes", + ) + ), + owned_concepts=_mapping_texts( + value.get("owned_concepts"), "owned concepts" + ), + non_owned_concepts=_mapping_texts( + value.get("non_owned_concepts"), "non-owned concepts" + ), + reference_packages=_mapping_texts( + value.get("reference_packages"), "reference packages" + ), + target_tested_providers=_mapping_texts( + value.get("target_tested_providers"), "target-tested providers" + ), + documentation=ModuleArchitectureDocumentation( + migration=_mapping_texts( + documentation.get("migration"), "migration documentation" + ), + upgrade=_mapping_texts( + documentation.get("upgrade"), "upgrade documentation" + ), + recovery=_mapping_texts( + documentation.get("recovery"), "recovery documentation" + ), + security=_mapping_texts( + documentation.get("security"), "security documentation" + ), + operations=_mapping_texts( + documentation.get("operations"), "operations documentation" + ), + ), + ) + + +def external_provider_from_mapping( + value: Mapping[str, object], +) -> ExternalProviderDeclaration: + raw_behavior = value.get("behavior") + if not isinstance(raw_behavior, Mapping): + raise ArchitectureDeclarationError( + "External provider behavior must be an object." + ) + return ExternalProviderDeclaration( + contract_version=str(value.get("contract_version") or "1"), + id=_mapping_text(value, "id"), + module_id=_mapping_text(value, "module_id"), + label=_mapping_text(value, "label"), + maturity=cast(IntegrationMaturity, _mapping_text(value, "maturity")), + operations=tuple( + cast(ProviderOperation, item) + for item in _mapping_texts(value.get("operations"), "provider operations") + ), + objects=tuple( + ProviderObjectDeclaration( + object_type=_mapping_text(item, "object_type"), + field_groups=_mapping_texts( + item.get("field_groups"), "provider field groups" + ), + authority_modes=tuple( + cast(SourceAuthorityMode, mode) + for mode in _mapping_texts( + item.get("authority_modes"), + "provider authority modes", + ) + ), + default_authority_mode=cast( + SourceAuthorityMode, + _mapping_text(item, "default_authority_mode"), + ), + ) + for item in _mapping_items(value.get("objects"), "provider objects") + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens=_mapping_optional_text( + raw_behavior.get("revision_tokens") + ), + concurrency=_mapping_optional_text(raw_behavior.get("concurrency")), + freshness=_mapping_optional_text(raw_behavior.get("freshness")), + health=_mapping_optional_text(raw_behavior.get("health")), + max_read_items=_mapping_optional_int( + raw_behavior.get("max_read_items") + ), + idempotency=_mapping_optional_text(raw_behavior.get("idempotency")), + retry=_mapping_optional_text(raw_behavior.get("retry")), + timeout_seconds=_mapping_optional_int( + raw_behavior.get("timeout_seconds") + ), + conflicts=_mapping_optional_text(raw_behavior.get("conflicts")), + outcome_unknown=_mapping_optional_text( + raw_behavior.get("outcome_unknown") + ), + outcome_unknown_supported=bool( + raw_behavior.get("outcome_unknown_supported", False) + ), + evidence=_mapping_optional_text(raw_behavior.get("evidence")), + audit_event_types=_mapping_texts( + raw_behavior.get("audit_event_types"), "provider audit event types" + ), + correction=_mapping_optional_text(raw_behavior.get("correction")), + rollback=_mapping_optional_text(raw_behavior.get("rollback")), + compensation=_mapping_optional_text(raw_behavior.get("compensation")), + reconciliation=_mapping_optional_text( + raw_behavior.get("reconciliation") + ), + outage=_mapping_optional_text(raw_behavior.get("outage")), + classifications=_mapping_texts( + raw_behavior.get("classifications"), "provider classifications" + ), + purposes=_mapping_texts( + raw_behavior.get("purposes"), "provider purposes" + ), + retention=_mapping_optional_text(raw_behavior.get("retention")), + secret_handling=_mapping_optional_text( + raw_behavior.get("secret_handling") + ), + ), + capability_names=_mapping_texts( + value.get("capability_names"), "provider capability names" + ), + interface_names=_mapping_texts( + value.get("interface_names"), "provider interface names" + ), + action_keys=_mapping_texts(value.get("action_keys"), "provider action keys"), + effect_keys=_mapping_texts(value.get("effect_keys"), "provider effect keys"), + operational_check_ids=_mapping_texts( + value.get("operational_check_ids"), "provider operational check ids" + ), + documentation_topic_ids=_mapping_texts( + value.get("documentation_topic_ids"), + "provider documentation topic ids", + ), + policy_hooks=_mapping_texts(value.get("policy_hooks"), "provider policy hooks"), + ownership_resource_types=_mapping_texts( + value.get("ownership_resource_types"), + "provider ownership resource types", + ), + ) + + +def _validate_provider_authority_modes( + declaration: ExternalProviderDeclaration, +) -> None: + operations = set(declaration.operations) + for item in declaration.objects: + for mode in item.authority_modes: + if mode == "external_mirror" and "read" not in operations: + raise ArchitectureDeclarationError( + "External mirror providers must support read operations." + ) + if mode == "governed_sync": + if "read" not in operations or not ( + operations & {"write", "synchronize"} + ): + raise ArchitectureDeclarationError( + "Governed-sync providers must support read and write/synchronize operations." + ) + if integration_maturity_rank( + declaration.maturity + ) < integration_maturity_rank("synchronize"): + raise ArchitectureDeclarationError( + "Governed-sync providers require synchronize maturity." + ) + if mode == "external_authoritative" and not ( + operations & {"link", "search", "read", "write", "publish", "synchronize"} + ): + raise ArchitectureDeclarationError( + "External-authoritative providers must expose an external-system operation." + ) + + +def _validate_provider_behavior( + declaration: ExternalProviderDeclaration, +) -> None: + operations = set(declaration.operations) + behavior = declaration.behavior + if operations & _READ_OPERATIONS: + _require_behavior_fields( + behavior, + "read-capable provider", + ("freshness", "health"), + ) + if behavior.max_read_items is None: + raise ArchitectureDeclarationError( + "Read-capable providers must declare max_read_items." + ) + if operations & _EFFECT_OPERATIONS: + _require_behavior_fields( + behavior, + "effect-capable provider", + ( + "idempotency", + "retry", + "concurrency", + "conflicts", + "outcome_unknown", + "evidence", + "correction", + "reconciliation", + "outage", + "secret_handling", + ), + ) + if behavior.timeout_seconds is None: + raise ArchitectureDeclarationError( + "Effect-capable providers must declare timeout_seconds." + ) + if not behavior.audit_event_types: + raise ArchitectureDeclarationError( + "Effect-capable providers must declare audit event types." + ) + if behavior.outcome_unknown_supported and not behavior.reconciliation: + raise ArchitectureDeclarationError( + "Outcome-unknown support requires a reconciliation path." + ) + if not behavior.classifications: + raise ArchitectureDeclarationError( + "Provider declarations must state data classifications." + ) + if not behavior.purposes: + raise ArchitectureDeclarationError( + "Provider declarations must state allowed purposes." + ) + _require_text(behavior.retention or "", "Provider retention behavior") + + +def _aggregate_provider_states( + states: Sequence[ExternalProviderRuntimeState], + *, + truncated: bool, +) -> dict[str, object]: + configured = tuple(item for item in states if item.configured) + active = tuple(item for item in configured if item.active) + observed = active or configured or tuple(states) + authority_modes = tuple( + sorted( + { + item.authority_mode + for item in configured + if item.authority_mode is not None + } + ) + ) + last_successes = tuple( + item.last_success_at + for item in configured + if item.last_success_at is not None + ) + binding_ref = configured[0].binding_ref if len(configured) == 1 else None + authority_mode = authority_modes[0] if len(authority_modes) == 1 else None + payload: dict[str, object] = { + "contract_version": "1", + "provider_id": states[0].provider_id, + "binding_ref": binding_ref, + "authority_mode": authority_mode, + "authority_modes": list(authority_modes), + "configured": bool(configured), + "active": bool(active), + "health": _worst_state( + (item.health for item in observed), + ("healthy", "inactive", "unknown", "warning", "error"), + ), + "freshness": _worst_state( + (item.freshness for item in observed), + ("current", "not_applicable", "unknown", "stale"), + ), + "conflict": _worst_state( + (item.conflict for item in observed), + ("clear", "not_applicable", "unknown", "pending", "blocked"), + ), + "recovery": _worst_state( + (item.recovery for item in observed), + ("ready", "not_applicable", "unsupported", "unknown", "attention"), + ), + "observed_at": max(item.observed_at for item in observed).isoformat(), + "last_success_at": ( + max(last_successes).isoformat() if last_successes else None + ), + "detail": ( + states[0].detail + if len(states) == 1 + else f"{len(configured)} configured bindings; {len(active)} active." + ), + "metrics": { + "binding_count": len(configured), + "active_binding_count": len(active), + "truncated": truncated, + }, + "bindings": [item.to_dict() for item in states if item.configured], + } + return payload + + +def _worst_state(values: Iterable[str], order: tuple[str, ...]) -> str: + ranks = {value: index for index, value in enumerate(order)} + normalized = tuple(values) + return max(normalized, key=lambda value: ranks.get(value, len(order))) + + +def _unconfigured_provider_state( + provider_id: str, +) -> ExternalProviderRuntimeState: + return ExternalProviderRuntimeState( + provider_id=provider_id, + observed_at=_runtime_now(), + configured=False, + active=False, + health="inactive", + freshness="not_applicable", + conflict="not_applicable", + recovery="not_applicable", + detail="No configured binding was found in this scope.", + ) + + +def _runtime_now() -> datetime: + return datetime.now(UTC) + + +def _validate_aware_datetime(value: datetime, label: str) -> None: + if value.tzinfo is None or value.utcoffset() is None: + raise ArchitectureDeclarationError(f"{label} must include a timezone.") + + +def _require_behavior_fields( + behavior: ProviderBehaviorDeclaration, + label: str, + fields: tuple[str, ...], +) -> None: + missing = [ + field_name + for field_name in fields + if not str(getattr(behavior, field_name) or "").strip() + ] + if missing: + raise ArchitectureDeclarationError( + f"{label.capitalize()} is missing behavior declarations: " + + ", ".join(missing) + ) + + +def _require_text(value: str, label: str) -> None: + if not str(value or "").strip(): + raise ArchitectureDeclarationError(f"{label} is required.") + + +def _validate_text_tuple( + values: tuple[str, ...], + label: str, + *, + require: bool = False, +) -> None: + if require and not values: + raise ArchitectureDeclarationError(f"{label.capitalize()} are required.") + if any(not str(value or "").strip() for value in values): + raise ArchitectureDeclarationError( + f"{label.capitalize()} must not contain empty values." + ) + if len(values) != len(set(values)): + raise ArchitectureDeclarationError( + f"{label.capitalize()} must not contain duplicates." + ) + + +def _mapping_text(value: Mapping[str, object], key: str) -> str: + result = _mapping_optional_text(value.get(key)) + if result is None: + raise ArchitectureDeclarationError(f"Architecture field {key!r} is required.") + return result + + +def _mapping_optional_text(value: object | None) -> str | None: + if value is None: + return None + result = str(value).strip() + return result or None + + +def _mapping_optional_int(value: object | None) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError) as exc: + raise ArchitectureDeclarationError( + f"Expected an integer architecture value, got {value!r}." + ) from exc + + +def _mapping_texts(value: object | None, label: str) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str) or not isinstance(value, Sequence): + raise ArchitectureDeclarationError(f"{label.capitalize()} must be a list.") + return tuple(str(item) for item in value) + + +def _mapping_items( + value: object | None, + label: str, +) -> tuple[Mapping[str, object], ...]: + if value is None: + return () + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ArchitectureDeclarationError(f"{label.capitalize()} must be a list.") + if any(not isinstance(item, Mapping) for item in value): + raise ArchitectureDeclarationError( + f"{label.capitalize()} entries must be objects." + ) + return cast(tuple[Mapping[str, object], ...], tuple(value)) + + +__all__ = [ + "ARCHITECTURE_LAYERS", + "MATURITY_EVIDENCE_KINDS", + "MODULE_KINDS", + "MODULE_MATURITY_ORDER", + "PROVIDER_OPERATIONS", + "ArchitectureDeclarationError", + "ArchitectureLayer", + "ExternalProviderDeclaration", + "ExternalProviderRuntimeState", + "ExternalProviderStateContext", + "ExternalProviderStateProvider", + "ExternalProviderStateProviderRegistration", + "MaturityEvidenceKind", + "ModuleArchitectureDeclaration", + "ModuleArchitectureDocumentation", + "ModuleKind", + "ModuleMaturity", + "ModuleMaturityEvidence", + "ProviderBehaviorDeclaration", + "ProviderConflictState", + "ProviderFreshnessState", + "ProviderHealthState", + "ProviderObjectDeclaration", + "ProviderOperation", + "ProviderRecoveryState", + "collect_external_provider_states", + "declared_module_architecture", + "module_architecture_issues", + "external_provider_from_mapping", + "module_architecture_from_mapping", +] diff --git a/src/govoplan_core/core/recovery.py b/src/govoplan_core/core/recovery.py new file mode 100644 index 0000000..710d4bf --- /dev/null +++ b/src/govoplan_core/core/recovery.py @@ -0,0 +1,669 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import StrEnum +import hashlib +import json +from typing import Any +from uuid import uuid4 + +from sqlalchemy import ( + BigInteger, + DateTime, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, + select, +) +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Mapped, Session, mapped_column + +from govoplan_core.core.runtime_coordination import LeaseClaim, assert_lease_fence +from govoplan_core.db.base import Base, TimestampMixin, utcnow +from govoplan_core.security.redaction import contains_plain_secret + + +class RecoveryMode(StrEnum): + ATOMIC = "atomic" + COMPENSATION = "compensation" + SNAPSHOT_RESTORE = "snapshot_restore" + FORWARD_RECOVERY = "forward_recovery" + IRREVERSIBLE = "irreversible" + + +class RecoveryStatus(StrEnum): + PLANNED = "planned" + PREPARED = "prepared" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + OUTCOME_UNKNOWN = "outcome_unknown" + RECOVERY_REQUIRED = "recovery_required" + RECOVERING = "recovering" + RECOVERED = "recovered" + MANUAL_INTERVENTION = "manual_intervention" + + +TERMINAL_RECOVERY_STATUSES = frozenset( + { + RecoveryStatus.SUCCEEDED.value, + RecoveryStatus.FAILED.value, + RecoveryStatus.RECOVERED.value, + RecoveryStatus.MANUAL_INTERVENTION.value, + } +) + + +_TRANSITIONS: dict[str, frozenset[str]] = { + RecoveryStatus.PLANNED.value: frozenset( + {RecoveryStatus.PREPARED.value, RecoveryStatus.FAILED.value} + ), + RecoveryStatus.PREPARED.value: frozenset( + {RecoveryStatus.RUNNING.value, RecoveryStatus.FAILED.value} + ), + RecoveryStatus.RUNNING.value: frozenset( + { + RecoveryStatus.SUCCEEDED.value, + RecoveryStatus.FAILED.value, + RecoveryStatus.OUTCOME_UNKNOWN.value, + RecoveryStatus.RECOVERY_REQUIRED.value, + } + ), + RecoveryStatus.OUTCOME_UNKNOWN.value: frozenset( + { + RecoveryStatus.SUCCEEDED.value, + RecoveryStatus.RECOVERY_REQUIRED.value, + RecoveryStatus.MANUAL_INTERVENTION.value, + } + ), + RecoveryStatus.RECOVERY_REQUIRED.value: frozenset( + { + RecoveryStatus.RECOVERING.value, + RecoveryStatus.MANUAL_INTERVENTION.value, + } + ), + RecoveryStatus.RECOVERING.value: frozenset( + { + RecoveryStatus.RECOVERED.value, + RecoveryStatus.MANUAL_INTERVENTION.value, + } + ), +} + + +class RecoveryGuaranteeError(ValueError): + pass + + +class RecoveryIdempotencyConflict(RecoveryGuaranteeError): + pass + + +class RecoveryOperation(Base, TimestampMixin): + __tablename__ = "core_recovery_operations" + __table_args__ = ( + UniqueConstraint( + "installation_id", + "module_id", + "idempotency_key", + name="uq_core_recovery_operation_idempotency", + ), + Index( + "ix_core_recovery_operations_status_updated", + "installation_id", + "status", + "updated_at", + ), + Index( + "ix_core_recovery_operations_resource", + "module_id", + "resource_type", + "resource_id", + ), + ) + + id: Mapped[str] = mapped_column( + String(36), + primary_key=True, + default=lambda: str(uuid4()), + ) + installation_id: Mapped[str] = mapped_column( + String(100), nullable=False, index=True + ) + module_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + operation_type: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + resource_type: Mapped[str | None] = mapped_column(String(100), index=True) + resource_id: Mapped[str | None] = mapped_column(String(255), index=True) + mode: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + status: Mapped[str] = mapped_column( + String(40), + default=RecoveryStatus.PLANNED.value, + nullable=False, + index=True, + ) + idempotency_key: Mapped[str] = mapped_column(String(200), nullable=False) + request_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + plan: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + backup_reference: Mapped[str | None] = mapped_column(String(1000)) + approval_reference: Mapped[str | None] = mapped_column(String(1000)) + lease_resource_key: Mapped[str | None] = mapped_column(String(255)) + holder_node_id: Mapped[str | None] = mapped_column(String(200)) + holder_incarnation: Mapped[str | None] = mapped_column(String(36)) + fencing_token: Mapped[int | None] = mapped_column( + BigInteger().with_variant(Integer, "sqlite") + ) + checkpoint_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + evidence_head_sha256: Mapped[str | None] = mapped_column(String(64)) + failure_summary: Mapped[str | None] = mapped_column(Text) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + recovery_started_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True) + ) + recovered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + metadata_: Mapped[dict[str, Any]] = mapped_column( + "metadata", JSON, default=dict, nullable=False + ) + + +class RecoveryCheckpoint(Base): + __tablename__ = "core_recovery_checkpoints" + __table_args__ = ( + UniqueConstraint( + "operation_id", + "sequence", + name="uq_core_recovery_checkpoint_sequence", + ), + Index( + "ix_core_recovery_checkpoints_operation_created", + "operation_id", + "created_at", + ), + ) + + id: Mapped[str] = mapped_column( + String(36), + primary_key=True, + default=lambda: str(uuid4()), + ) + operation_id: Mapped[str] = mapped_column( + ForeignKey("core_recovery_operations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + sequence: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + kind: Mapped[str] = mapped_column(String(80), nullable=False) + summary: Mapped[str] = mapped_column(Text, nullable=False) + evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + previous_sha256: Mapped[str | None] = mapped_column(String(64)) + checkpoint_sha256: Mapped[str] = mapped_column( + String(64), nullable=False, index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, nullable=False + ) + + +@dataclass(frozen=True, slots=True) +class RecoveryPlan: + mode: RecoveryMode + preconditions: tuple[str, ...] = () + compensation_steps: tuple[str, ...] = () + forward_recovery_steps: tuple[str, ...] = () + verification_steps: tuple[str, ...] = () + backup_reference: str | None = None + approval_reference: str | None = None + + def validate(self) -> None: + if not self.verification_steps: + raise RecoveryGuaranteeError( + "Recovery plans require at least one verification step" + ) + if self.mode == RecoveryMode.COMPENSATION and not self.compensation_steps: + raise RecoveryGuaranteeError( + "Compensation recovery requires explicit compensation steps" + ) + if self.mode == RecoveryMode.SNAPSHOT_RESTORE and not self.backup_reference: + raise RecoveryGuaranteeError( + "Snapshot restore requires a verified backup reference" + ) + if ( + self.mode == RecoveryMode.FORWARD_RECOVERY + and not self.forward_recovery_steps + ): + raise RecoveryGuaranteeError( + "Forward recovery requires explicit forward-recovery steps" + ) + if self.mode == RecoveryMode.IRREVERSIBLE and not self.approval_reference: + raise RecoveryGuaranteeError( + "Irreversible operations require an approval reference" + ) + + def as_dict(self) -> dict[str, Any]: + return { + "mode": self.mode.value, + "preconditions": list(self.preconditions), + "compensation_steps": list(self.compensation_steps), + "forward_recovery_steps": list(self.forward_recovery_steps), + "verification_steps": list(self.verification_steps), + "backup_reference": self.backup_reference, + "approval_reference": self.approval_reference, + } + + +def plan_recovery_operation( + session: Session, + *, + installation_id: str, + module_id: str, + operation_type: str, + idempotency_key: str, + request: dict[str, Any], + recovery_plan: RecoveryPlan, + resource_type: str | None = None, + resource_id: str | None = None, + lease_claim: LeaseClaim | None = None, + metadata: dict[str, Any] | None = None, +) -> RecoveryOperation: + recovery_plan.validate() + request_sha256 = _canonical_sha256(request) + existing = session.execute( + select(RecoveryOperation).where( + RecoveryOperation.installation_id == installation_id, + RecoveryOperation.module_id == module_id, + RecoveryOperation.idempotency_key == idempotency_key, + ) + ).scalar_one_or_none() + if existing is not None: + if existing.request_sha256 != request_sha256: + raise RecoveryIdempotencyConflict( + "Recovery operation idempotency key was reused for another request" + ) + return existing + if contains_plain_secret(metadata or {}): + raise RecoveryGuaranteeError( + "Recovery metadata must contain secret references, not plaintext secrets" + ) + if lease_claim is not None: + if lease_claim.installation_id != installation_id: + raise RecoveryGuaranteeError( + "Recovery operation and lease belong to different installations" + ) + assert_lease_fence(session, lease_claim) + operation = RecoveryOperation( + installation_id=installation_id, + module_id=module_id, + operation_type=operation_type, + resource_type=resource_type, + resource_id=resource_id, + mode=recovery_plan.mode.value, + status=RecoveryStatus.PLANNED.value, + idempotency_key=idempotency_key, + request_sha256=request_sha256, + plan=recovery_plan.as_dict(), + backup_reference=recovery_plan.backup_reference, + approval_reference=recovery_plan.approval_reference, + lease_resource_key=lease_claim.resource_key if lease_claim else None, + holder_node_id=lease_claim.holder_node_id if lease_claim else None, + holder_incarnation=lease_claim.holder_incarnation if lease_claim else None, + fencing_token=lease_claim.fencing_token if lease_claim else None, + metadata_=dict(metadata or {}), + ) + try: + with session.begin_nested(): + session.add(operation) + session.flush() + except IntegrityError: + existing = session.execute( + select(RecoveryOperation).where( + RecoveryOperation.installation_id == installation_id, + RecoveryOperation.module_id == module_id, + RecoveryOperation.idempotency_key == idempotency_key, + ) + ).scalar_one_or_none() + if existing is None: + raise + if existing.request_sha256 != request_sha256: + raise RecoveryIdempotencyConflict( + "Recovery operation idempotency key was reused for another request" + ) + return existing + record_recovery_checkpoint( + session, + operation, + kind="plan", + summary="Recovery contract recorded before side effects", + evidence={"request_sha256": request_sha256, "plan": recovery_plan.as_dict()}, + lease_claim=lease_claim, + ) + return operation + + +def prepare_recovery_operation( + session: Session, + operation: RecoveryOperation, + *, + evidence: dict[str, Any], + lease_claim: LeaseClaim | None = None, +) -> RecoveryOperation: + _verify_operation_fence(session, operation, lease_claim) + if not evidence: + raise RecoveryGuaranteeError( + "Recovery preparation requires durable precondition evidence" + ) + return transition_recovery_operation( + session, + operation, + status=RecoveryStatus.PREPARED, + kind="prepared", + summary="Preconditions and recovery material verified", + evidence=evidence, + lease_claim=lease_claim, + ) + + +def start_recovery_operation( + session: Session, + operation: RecoveryOperation, + *, + evidence: dict[str, Any] | None = None, + lease_claim: LeaseClaim | None = None, + now: datetime | None = None, +) -> RecoveryOperation: + _verify_operation_fence(session, operation, lease_claim) + operation.started_at = _as_utc(now or utcnow()) + return transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RUNNING, + kind="started", + summary="Guarded operation started", + evidence=evidence or {}, + lease_claim=lease_claim, + ) + + +def transition_recovery_operation( + session: Session, + operation: RecoveryOperation, + *, + status: RecoveryStatus, + kind: str, + summary: str, + evidence: dict[str, Any] | None = None, + failure_summary: str | None = None, + lease_claim: LeaseClaim | None = None, + now: datetime | None = None, +) -> RecoveryOperation: + locked = session.execute( + select(RecoveryOperation) + .where(RecoveryOperation.id == operation.id) + .with_for_update() + ).scalar_one() + _verify_operation_fence(session, locked, lease_claim) + allowed = _TRANSITIONS.get(locked.status, frozenset()) + if status.value not in allowed: + raise RecoveryGuaranteeError( + f"Recovery transition {locked.status!r} -> {status.value!r} is not allowed" + ) + _validate_mode_transition(locked, status) + _validate_transition_evidence( + status=status, + evidence=evidence or {}, + failure_summary=failure_summary, + ) + observed_at = _as_utc(now or utcnow()) + locked.status = status.value + locked.revision = int(locked.revision or 0) + 1 + if failure_summary is not None: + locked.failure_summary = failure_summary + if status == RecoveryStatus.SUCCEEDED: + locked.completed_at = observed_at + elif status == RecoveryStatus.RECOVERING: + locked.recovery_started_at = observed_at + elif status == RecoveryStatus.RECOVERED: + locked.recovered_at = observed_at + locked.completed_at = observed_at + elif status in {RecoveryStatus.FAILED, RecoveryStatus.MANUAL_INTERVENTION}: + locked.completed_at = observed_at + session.add(locked) + record_recovery_checkpoint( + session, + locked, + kind=kind, + summary=summary, + evidence=evidence or {}, + lease_claim=lease_claim, + now=observed_at, + ) + session.flush() + return locked + + +def record_recovery_checkpoint( + session: Session, + operation: RecoveryOperation, + *, + kind: str, + summary: str, + evidence: dict[str, Any], + lease_claim: LeaseClaim | None = None, + now: datetime | None = None, +) -> RecoveryCheckpoint: + if contains_plain_secret(evidence): + raise RecoveryGuaranteeError( + "Recovery evidence must contain secret references, not plaintext secrets" + ) + locked = session.execute( + select(RecoveryOperation) + .where(RecoveryOperation.id == operation.id) + .with_for_update() + ).scalar_one() + _verify_operation_fence(session, locked, lease_claim) + observed_at = _as_utc(now or utcnow()) + sequence = int(locked.checkpoint_count or 0) + 1 + payload = { + "operation_id": locked.id, + "sequence": sequence, + "status": locked.status, + "kind": kind, + "summary": summary, + "evidence": evidence, + "previous_sha256": locked.evidence_head_sha256, + "created_at": observed_at.isoformat(), + } + checkpoint_hash = _canonical_sha256(payload) + checkpoint = RecoveryCheckpoint( + operation_id=locked.id, + sequence=sequence, + status=locked.status, + kind=kind, + summary=summary, + evidence=dict(evidence), + previous_sha256=locked.evidence_head_sha256, + checkpoint_sha256=checkpoint_hash, + created_at=observed_at, + ) + locked.checkpoint_count = sequence + locked.evidence_head_sha256 = checkpoint_hash + session.add(locked) + session.add(checkpoint) + session.flush() + return checkpoint + + +def verify_recovery_evidence_chain( + session: Session, + operation_id: str, +) -> bool: + operation = session.get(RecoveryOperation, operation_id) + if operation is None: + raise RecoveryGuaranteeError("Recovery operation was not found") + checkpoints = ( + session.execute( + select(RecoveryCheckpoint) + .where(RecoveryCheckpoint.operation_id == operation_id) + .order_by(RecoveryCheckpoint.sequence) + ) + .scalars() + .all() + ) + previous: str | None = None + for expected_sequence, checkpoint in enumerate(checkpoints, start=1): + if ( + checkpoint.sequence != expected_sequence + or checkpoint.previous_sha256 != previous + ): + return False + payload = { + "operation_id": checkpoint.operation_id, + "sequence": checkpoint.sequence, + "status": checkpoint.status, + "kind": checkpoint.kind, + "summary": checkpoint.summary, + "evidence": checkpoint.evidence, + "previous_sha256": checkpoint.previous_sha256, + "created_at": _as_utc(checkpoint.created_at).isoformat(), + } + if _canonical_sha256(payload) != checkpoint.checkpoint_sha256: + return False + previous = checkpoint.checkpoint_sha256 + return ( + len(checkpoints) == int(operation.checkpoint_count or 0) + and previous == operation.evidence_head_sha256 + ) + + +def operation_recovery_action(mode: RecoveryMode) -> RecoveryStatus: + if mode == RecoveryMode.ATOMIC: + return RecoveryStatus.FAILED + if mode in { + RecoveryMode.COMPENSATION, + RecoveryMode.SNAPSHOT_RESTORE, + RecoveryMode.FORWARD_RECOVERY, + }: + return RecoveryStatus.RECOVERY_REQUIRED + return RecoveryStatus.MANUAL_INTERVENTION + + +def _validate_mode_transition( + operation: RecoveryOperation, + status: RecoveryStatus, +) -> None: + mode = RecoveryMode(operation.mode) + if ( + operation.status == RecoveryStatus.RUNNING.value + and status == RecoveryStatus.FAILED + and mode != RecoveryMode.ATOMIC + ): + raise RecoveryGuaranteeError( + "A non-atomic operation cannot be marked failed after it starts; " + "record recovery_required, outcome_unknown, or manual_intervention" + ) + if mode == RecoveryMode.ATOMIC and status in { + RecoveryStatus.RECOVERY_REQUIRED, + RecoveryStatus.RECOVERING, + RecoveryStatus.RECOVERED, + }: + raise RecoveryGuaranteeError( + "Atomic operations must roll back in their transaction instead of entering recovery" + ) + if mode == RecoveryMode.IRREVERSIBLE and status in { + RecoveryStatus.RECOVERY_REQUIRED, + RecoveryStatus.RECOVERING, + RecoveryStatus.RECOVERED, + }: + raise RecoveryGuaranteeError( + "Irreversible operations cannot claim automated recovery" + ) + + +def _validate_transition_evidence( + *, + status: RecoveryStatus, + evidence: dict[str, Any], + failure_summary: str | None, +) -> None: + if status in {RecoveryStatus.SUCCEEDED, RecoveryStatus.RECOVERED}: + checks = evidence.get("checks") + if ( + evidence.get("verified") is not True + or not isinstance( + checks, + (dict, list), + ) + or not checks + ): + raise RecoveryGuaranteeError( + "Successful recovery transitions require verified evidence and check results" + ) + if status == RecoveryStatus.MANUAL_INTERVENTION and not failure_summary: + raise RecoveryGuaranteeError( + "Manual intervention requires an operator-facing failure summary" + ) + + +def _verify_operation_fence( + session: Session, + operation: RecoveryOperation, + lease_claim: LeaseClaim | None, +) -> None: + if operation.lease_resource_key is None: + return + if lease_claim is None: + raise RecoveryGuaranteeError( + "This recovery operation requires its distributed lease fence" + ) + if ( + lease_claim.resource_key != operation.lease_resource_key + or lease_claim.holder_node_id != operation.holder_node_id + or lease_claim.holder_incarnation != operation.holder_incarnation + or lease_claim.fencing_token != operation.fencing_token + ): + raise RecoveryGuaranteeError( + "Recovery operation lease does not match its recorded fence" + ) + assert_lease_fence(session, lease_claim) + + +def _canonical_sha256(value: dict[str, Any]) -> str: + encoded = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _as_utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +__all__ = [ + "RecoveryCheckpoint", + "RecoveryGuaranteeError", + "RecoveryIdempotencyConflict", + "RecoveryMode", + "RecoveryOperation", + "RecoveryPlan", + "RecoveryStatus", + "TERMINAL_RECOVERY_STATUSES", + "operation_recovery_action", + "plan_recovery_operation", + "prepare_recovery_operation", + "record_recovery_checkpoint", + "start_recovery_operation", + "transition_recovery_operation", + "verify_recovery_evidence_chain", +] diff --git a/src/govoplan_core/core/registry.py b/src/govoplan_core/core/registry.py index 461644c..5c14cf8 100644 --- a/src/govoplan_core/core/registry.py +++ b/src/govoplan_core/core/registry.py @@ -29,6 +29,12 @@ from govoplan_core.core.ownership import ( OwnershipProviderRegistration, ResourceOwnershipProvider, ) +from govoplan_core.core.provider_governance import ( + ExternalProviderDeclaration, + ExternalProviderStateProviderRegistration, + ModuleArchitectureDeclaration, + module_architecture_issues, +) from govoplan_core.core.versioning import format_version_range, version_range_is_valid, version_satisfies_range from govoplan_core.core.search import ( RegisteredSearchProvider, @@ -163,6 +169,33 @@ class PlatformRegistry: def manifests(self) -> tuple[ModuleManifest, ...]: return tuple(self._topologically_sorted()) + def module_architectures( + self, + ) -> tuple[tuple[str, ModuleArchitectureDeclaration], ...]: + return tuple( + (manifest.id, manifest.architecture) + for manifest in self.manifests() + if manifest.architecture is not None + ) + + def external_provider_declarations( + self, + ) -> tuple[ExternalProviderDeclaration, ...]: + return tuple( + declaration + for manifest in self.manifests() + for declaration in manifest.external_providers + ) + + def external_provider_state_providers( + self, + ) -> tuple[ExternalProviderStateProviderRegistration, ...]: + return tuple( + registration + for manifest in self.manifests() + for registration in manifest.external_provider_state_providers + ) + def permissions(self) -> tuple[PermissionDefinition, ...]: return tuple(permission for manifest in self.manifests() for permission in manifest.permissions) @@ -629,9 +662,160 @@ def _validate_manifest_shape(manifest: ModuleManifest) -> None: f"Module {manifest.id!r} documentation topic {topic.id!r}: {issue}" ) _validate_documentation_extensions(manifest) + _validate_architecture_declarations(manifest) _validate_workflow_definition_contributions(manifest) +def _validate_architecture_declarations(manifest: ModuleManifest) -> None: + architecture = manifest.architecture + if architecture is not None: + for issue in module_architecture_issues( + architecture, + has_migrations=manifest.migration_spec is not None, + ): + raise RegistryError( + f"Module {manifest.id!r} architecture declaration: {issue}" + ) + + provider_ids: set[str] = set() + declared_capabilities = { + *manifest.required_capabilities, + *manifest.optional_capabilities, + *manifest.capability_factories, + } + declared_interfaces = { + *(item.name for item in manifest.provides_interfaces), + *(item.name for item in manifest.requires_interfaces), + } + operational_check_ids = { + item.check_id for item in manifest.operational_check_providers + } + documentation_topic_ids = {item.id for item in manifest.documentation} + ownership_resource_types = { + item.resource_type for item in manifest.ownership_providers + } + provider_authority_modes: set[str] = set() + + for declaration in manifest.external_providers: + if declaration.module_id != manifest.id: + raise RegistryError( + f"Provider declaration {declaration.id!r} belongs to " + f"{declaration.module_id!r}, not module {manifest.id!r}" + ) + if declaration.id in provider_ids: + raise RegistryError( + f"Module {manifest.id!r} declares duplicate external provider " + f"{declaration.id!r}" + ) + provider_ids.add(declaration.id) + provider_authority_modes.update(declaration.authority_modes) + _validate_provider_references( + manifest.id, + declaration, + declared_capabilities=declared_capabilities, + declared_interfaces=declared_interfaces, + operational_check_ids=operational_check_ids, + documentation_topic_ids=documentation_topic_ids, + ownership_resource_types=ownership_resource_types, + ) + + state_provider_ids: set[str] = set() + for registration in manifest.external_provider_state_providers: + if registration.module_id != manifest.id: + raise RegistryError( + f"Provider state registration {registration.provider_id!r} belongs " + f"to {registration.module_id!r}, not module {manifest.id!r}" + ) + if registration.provider_id not in provider_ids: + raise RegistryError( + f"Module {manifest.id!r} registers runtime state for undeclared " + f"provider {registration.provider_id!r}" + ) + if registration.provider_id in state_provider_ids: + raise RegistryError( + f"Module {manifest.id!r} registers duplicate runtime state for " + f"provider {registration.provider_id!r}" + ) + state_provider_ids.add(registration.provider_id) + + missing_state_provider_ids = provider_ids - state_provider_ids + if missing_state_provider_ids: + raise RegistryError( + f"Module {manifest.id!r} external providers require sanitized runtime " + "state providers: " + ", ".join(sorted(missing_state_provider_ids)) + ) + + if architecture is None: + if manifest.external_providers or manifest.external_provider_state_providers: + raise RegistryError( + f"Module {manifest.id!r} declares external providers without a " + "module architecture declaration" + ) + return + missing_modes = provider_authority_modes - set( + architecture.supported_authority_modes + ) + if missing_modes: + raise RegistryError( + f"Module {manifest.id!r} provider authority modes are not declared " + "by its architecture metadata: " + ", ".join(sorted(missing_modes)) + ) + unknown_target_providers = set( + architecture.target_tested_providers + ) - provider_ids + if unknown_target_providers: + raise RegistryError( + f"Module {manifest.id!r} target-tested providers are not declared: " + + ", ".join(sorted(unknown_target_providers)) + ) + + +def _validate_provider_references( + module_id: str, + declaration: ExternalProviderDeclaration, + *, + declared_capabilities: set[str], + declared_interfaces: set[str], + operational_check_ids: set[str], + documentation_topic_ids: set[str], + ownership_resource_types: set[str], +) -> None: + references = ( + ( + "capabilities", + set(declaration.capability_names), + declared_capabilities, + ), + ( + "interfaces", + set(declaration.interface_names), + declared_interfaces, + ), + ( + "operational checks", + set(declaration.operational_check_ids), + operational_check_ids, + ), + ( + "documentation topics", + set(declaration.documentation_topic_ids), + documentation_topic_ids, + ), + ( + "ownership resource types", + set(declaration.ownership_resource_types), + ownership_resource_types, + ), + ) + for label, requested, available in references: + missing = requested - available + if missing: + raise RegistryError( + f"Module {module_id!r} provider {declaration.id!r} references " + f"undeclared {label}: " + ", ".join(sorted(missing)) + ) + + def _validate_workflow_definition_contributions( manifest: ModuleManifest, ) -> None: diff --git a/src/govoplan_core/core/runtime_coordination.py b/src/govoplan_core/core/runtime_coordination.py new file mode 100644 index 0000000..629b09e --- /dev/null +++ b/src/govoplan_core/core/runtime_coordination.py @@ -0,0 +1,605 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from enum import StrEnum +import hashlib +import json +import os +import socket +from typing import Any +from uuid import uuid4 + +from sqlalchemy import BigInteger, DateTime, Index, Integer, JSON, String, UniqueConstraint, select +from sqlalchemy.orm import Mapped, Session, mapped_column + +from govoplan_core.db.base import Base, TimestampMixin, utcnow + + +class RuntimeNodeState(StrEnum): + ACTIVE = "active" + DRAINING = "draining" + STOPPED = "stopped" + + +class RuntimeCoordinationError(RuntimeError): + pass + + +class LeaseUnavailable(RuntimeCoordinationError): + pass + + +class StaleFence(RuntimeCoordinationError): + pass + + +class RuntimeNode(Base, TimestampMixin): + __tablename__ = "core_runtime_nodes" + __table_args__ = ( + UniqueConstraint( + "installation_id", + "node_id", + name="uq_core_runtime_node_installation_node", + ), + Index( + "ix_core_runtime_nodes_installation_state_heartbeat", + "installation_id", + "state", + "last_heartbeat_at", + ), + ) + + id: Mapped[str] = mapped_column( + String(36), + primary_key=True, + default=lambda: str(uuid4()), + ) + installation_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + node_id: Mapped[str] = mapped_column(String(200), nullable=False, index=True) + incarnation: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + role: Mapped[str] = mapped_column(String(40), nullable=False, index=True) + software_version: Mapped[str] = mapped_column(String(80), nullable=False) + composition_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + queues: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + state: Mapped[str] = mapped_column( + String(30), + default=RuntimeNodeState.ACTIVE.value, + nullable=False, + index=True, + ) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) + last_heartbeat_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) + drain_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + drain_reason: Mapped[str | None] = mapped_column(String(500)) + stopped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False) + + +class DistributedLease(Base, TimestampMixin): + __tablename__ = "core_distributed_leases" + __table_args__ = ( + UniqueConstraint( + "installation_id", + "resource_key", + name="uq_core_distributed_lease_resource", + ), + Index( + "ix_core_distributed_leases_expiry", + "installation_id", + "expires_at", + ), + ) + + id: Mapped[str] = mapped_column( + String(36), + primary_key=True, + default=lambda: str(uuid4()), + ) + installation_id: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + resource_key: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + holder_node_id: Mapped[str | None] = mapped_column(String(200), index=True) + holder_incarnation: Mapped[str | None] = mapped_column(String(36), index=True) + fencing_token: Mapped[int] = mapped_column( + BigInteger().with_variant(Integer, "sqlite"), + default=0, + nullable=False, + ) + acquired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + renewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) + metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False) + + +@dataclass(frozen=True, slots=True) +class RuntimeIdentity: + installation_id: str + node_id: str + incarnation: str + role: str + software_version: str + composition_hash: str + queues: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class LeaseClaim: + installation_id: str + resource_key: str + holder_node_id: str + holder_incarnation: str + fencing_token: int + expires_at: datetime + + +def runtime_identity( + settings: object, + *, + software_version: str, + module_ids: tuple[str, ...] = (), + role: str | None = None, + node_id: str | None = None, + incarnation: str | None = None, + queues: tuple[str, ...] | None = None, +) -> RuntimeIdentity: + effective_role = str( + role or getattr(settings, "runtime_role", "api") or "api" + ).strip().lower() + effective_node_id = str( + node_id + or getattr(settings, "runtime_node_id", None) + or os.getenv("HOSTNAME") + or socket.gethostname() + ).strip() + installation_id = str( + getattr(settings, "installation_id", "govoplan-local") + or "govoplan-local" + ).strip() + if not installation_id or not effective_node_id or not effective_role: + raise RuntimeCoordinationError( + "Runtime identity requires installation, node, and role identifiers" + ) + effective_queues = queues + if effective_queues is None: + effective_queues = tuple( + item.strip() + for item in str(getattr(settings, "celery_queues", "") or "").split(",") + if item.strip() + ) + composition_hash = hashlib.sha256( + json.dumps(sorted(module_ids), separators=(",", ":")).encode("utf-8") + ).hexdigest() + return RuntimeIdentity( + installation_id=installation_id, + node_id=effective_node_id, + incarnation=incarnation or str(uuid4()), + role=effective_role, + software_version=str(software_version), + composition_hash=composition_hash, + queues=tuple(sorted(set(effective_queues))), + ) + + +def register_runtime_node( + session: Session, + identity: RuntimeIdentity, + *, + metadata: dict[str, Any] | None = None, + now: datetime | None = None, +) -> RuntimeNode: + observed_at = _as_utc(now or utcnow()) + _insert_node_placeholder(session, identity, observed_at) + node = _locked_node(session, identity.installation_id, identity.node_id) + if node is None: # pragma: no cover - guarded by insert/upsert + raise RuntimeCoordinationError("Runtime node registration was not persisted") + node.incarnation = identity.incarnation + node.role = identity.role + node.software_version = identity.software_version + node.composition_hash = identity.composition_hash + node.queues = list(identity.queues) + node.state = RuntimeNodeState.ACTIVE.value + node.started_at = observed_at + node.last_heartbeat_at = observed_at + node.drain_requested_at = None + node.drain_reason = None + node.stopped_at = None + node.metadata_ = dict(metadata or {}) + session.add(node) + session.flush() + return node + + +def heartbeat_runtime_node( + session: Session, + identity: RuntimeIdentity, + *, + now: datetime | None = None, + metadata: dict[str, Any] | None = None, +) -> RuntimeNode: + node = _locked_node(session, identity.installation_id, identity.node_id) + if node is None or node.incarnation != identity.incarnation: + raise RuntimeCoordinationError( + "Runtime heartbeat was rejected for a stale node incarnation" + ) + if node.state == RuntimeNodeState.STOPPED.value: + raise RuntimeCoordinationError("Stopped runtime node cannot heartbeat") + node.last_heartbeat_at = _as_utc(now or utcnow()) + if metadata is not None: + node.metadata_ = dict(metadata) + session.add(node) + session.flush() + return node + + +def request_runtime_node_drain( + session: Session, + *, + installation_id: str, + node_id: str, + reason: str | None = None, + now: datetime | None = None, +) -> RuntimeNode: + node = _locked_node(session, installation_id, node_id) + if node is None: + raise RuntimeCoordinationError("Runtime node was not found") + if node.state == RuntimeNodeState.STOPPED.value: + raise RuntimeCoordinationError("Stopped runtime node cannot be drained") + node.state = RuntimeNodeState.DRAINING.value + node.drain_requested_at = _as_utc(now or utcnow()) + node.drain_reason = str(reason or "operator request")[:500] + session.add(node) + session.flush() + return node + + +def cancel_runtime_node_drain( + session: Session, + *, + installation_id: str, + node_id: str, +) -> RuntimeNode: + node = _locked_node(session, installation_id, node_id) + if node is None: + raise RuntimeCoordinationError("Runtime node was not found") + if node.state != RuntimeNodeState.DRAINING.value: + return node + node.state = RuntimeNodeState.ACTIVE.value + node.drain_requested_at = None + node.drain_reason = None + session.add(node) + session.flush() + return node + + +def stop_runtime_node( + session: Session, + identity: RuntimeIdentity, + *, + now: datetime | None = None, +) -> bool: + node = _locked_node(session, identity.installation_id, identity.node_id) + if node is None or node.incarnation != identity.incarnation: + return False + observed_at = _as_utc(now or utcnow()) + node.state = RuntimeNodeState.STOPPED.value + node.stopped_at = observed_at + node.last_heartbeat_at = observed_at + session.add(node) + session.flush() + return True + + +def runtime_node_is_draining( + session: Session, + identity: RuntimeIdentity, +) -> bool: + node = session.execute( + select(RuntimeNode).where( + RuntimeNode.installation_id == identity.installation_id, + RuntimeNode.node_id == identity.node_id, + RuntimeNode.incarnation == identity.incarnation, + ) + ).scalar_one_or_none() + return node is not None and node.state == RuntimeNodeState.DRAINING.value + + +def list_runtime_nodes( + session: Session, + *, + installation_id: str, + stale_after_seconds: int, + now: datetime | None = None, +) -> list[dict[str, Any]]: + observed_at = _as_utc(now or utcnow()) + stale_before = observed_at - timedelta(seconds=max(1, stale_after_seconds)) + nodes = session.execute( + select(RuntimeNode) + .where(RuntimeNode.installation_id == installation_id) + .order_by(RuntimeNode.role, RuntimeNode.node_id) + ).scalars().all() + return [ + { + "node_id": node.node_id, + "incarnation": node.incarnation, + "role": node.role, + "software_version": node.software_version, + "composition_hash": node.composition_hash, + "queues": list(node.queues or []), + "state": node.state, + "started_at": _iso(node.started_at), + "last_heartbeat_at": _iso(node.last_heartbeat_at), + "drain_requested_at": _iso(node.drain_requested_at), + "drain_reason": node.drain_reason, + "stopped_at": _iso(node.stopped_at), + "stale": ( + node.state != RuntimeNodeState.STOPPED.value + and _as_utc(node.last_heartbeat_at) < stale_before + ), + "metadata": dict(node.metadata_ or {}), + } + for node in nodes + ] + + +def acquire_lease( + session: Session, + *, + installation_id: str, + resource_key: str, + holder_node_id: str, + holder_incarnation: str, + ttl_seconds: int, + metadata: dict[str, Any] | None = None, + now: datetime | None = None, +) -> LeaseClaim | None: + if ttl_seconds < 1: + raise ValueError("Lease TTL must be at least one second") + observed_at = _as_utc(now or utcnow()) + _insert_lease_placeholder(session, installation_id, resource_key, observed_at) + lease = _locked_lease(session, installation_id, resource_key) + if lease is None: # pragma: no cover - guarded by insert/upsert + raise RuntimeCoordinationError("Distributed lease was not persisted") + same_holder = ( + lease.holder_node_id == holder_node_id + and lease.holder_incarnation == holder_incarnation + ) + expired = _as_utc(lease.expires_at) <= observed_at + if lease.holder_node_id is not None and not same_holder and not expired: + return None + if not same_holder or expired: + lease.fencing_token = int(lease.fencing_token or 0) + 1 + lease.acquired_at = observed_at + lease.holder_node_id = holder_node_id + lease.holder_incarnation = holder_incarnation + lease.renewed_at = observed_at + lease.expires_at = observed_at + timedelta(seconds=ttl_seconds) + lease.metadata_ = dict(metadata or {}) + session.add(lease) + session.flush() + return _lease_claim(lease) + + +def renew_lease( + session: Session, + claim: LeaseClaim, + *, + ttl_seconds: int, + now: datetime | None = None, +) -> LeaseClaim: + if ttl_seconds < 1: + raise ValueError("Lease TTL must be at least one second") + observed_at = _as_utc(now or utcnow()) + lease = _locked_lease(session, claim.installation_id, claim.resource_key) + _assert_matching_fence(lease, claim, observed_at=observed_at) + lease.renewed_at = observed_at + lease.expires_at = observed_at + timedelta(seconds=ttl_seconds) + session.add(lease) + session.flush() + return _lease_claim(lease) + + +def release_lease( + session: Session, + claim: LeaseClaim, + *, + now: datetime | None = None, +) -> None: + observed_at = _as_utc(now or utcnow()) + lease = _locked_lease(session, claim.installation_id, claim.resource_key) + _assert_matching_fence(lease, claim, observed_at=observed_at) + lease.holder_node_id = None + lease.holder_incarnation = None + lease.renewed_at = observed_at + lease.expires_at = observed_at + session.add(lease) + session.flush() + + +def assert_lease_fence( + session: Session, + claim: LeaseClaim, + *, + now: datetime | None = None, +) -> None: + lease = _locked_lease(session, claim.installation_id, claim.resource_key) + _assert_matching_fence( + lease, + claim, + observed_at=_as_utc(now or utcnow()), + ) + + +def _insert_node_placeholder( + session: Session, + identity: RuntimeIdentity, + observed_at: datetime, +) -> None: + values = { + "id": str(uuid4()), + "installation_id": identity.installation_id, + "node_id": identity.node_id, + "incarnation": identity.incarnation, + "role": identity.role, + "software_version": identity.software_version, + "composition_hash": identity.composition_hash, + "queues": list(identity.queues), + "state": RuntimeNodeState.ACTIVE.value, + "started_at": observed_at, + "last_heartbeat_at": observed_at, + "metadata": {}, + "created_at": observed_at, + "updated_at": observed_at, + } + _insert_do_nothing( + session, + RuntimeNode.__table__, + values, + conflict_columns=("installation_id", "node_id"), + ) + + +def _insert_lease_placeholder( + session: Session, + installation_id: str, + resource_key: str, + observed_at: datetime, +) -> None: + values = { + "id": str(uuid4()), + "installation_id": installation_id, + "resource_key": resource_key, + "fencing_token": 0, + "expires_at": observed_at, + "metadata": {}, + "created_at": observed_at, + "updated_at": observed_at, + } + _insert_do_nothing( + session, + DistributedLease.__table__, + values, + conflict_columns=("installation_id", "resource_key"), + ) + + +def _insert_do_nothing( + session: Session, + table: Any, + values: dict[str, Any], + *, + conflict_columns: tuple[str, ...], +) -> None: + dialect = session.get_bind().dialect.name + if dialect == "postgresql": + from sqlalchemy.dialects.postgresql import insert + + statement = insert(table).values(**values).on_conflict_do_nothing( + index_elements=list(conflict_columns) + ) + elif dialect == "sqlite": + from sqlalchemy.dialects.sqlite import insert + + statement = insert(table).values(**values).on_conflict_do_nothing( + index_elements=list(conflict_columns) + ) + else: # pragma: no cover - GovOPlaN supports PostgreSQL and dev SQLite + raise RuntimeCoordinationError( + f"Distributed coordination is unsupported on {dialect!r}" + ) + session.execute(statement) + session.flush() + + +def _locked_node( + session: Session, + installation_id: str, + node_id: str, +) -> RuntimeNode | None: + return session.execute( + select(RuntimeNode) + .where( + RuntimeNode.installation_id == installation_id, + RuntimeNode.node_id == node_id, + ) + .with_for_update() + ).scalar_one_or_none() + + +def _locked_lease( + session: Session, + installation_id: str, + resource_key: str, +) -> DistributedLease | None: + return session.execute( + select(DistributedLease) + .where( + DistributedLease.installation_id == installation_id, + DistributedLease.resource_key == resource_key, + ) + .with_for_update() + ).scalar_one_or_none() + + +def _assert_matching_fence( + lease: DistributedLease | None, + claim: LeaseClaim, + *, + observed_at: datetime, +) -> None: + if ( + lease is None + or lease.holder_node_id != claim.holder_node_id + or lease.holder_incarnation != claim.holder_incarnation + or int(lease.fencing_token) != claim.fencing_token + or _as_utc(lease.expires_at) <= observed_at + ): + raise StaleFence( + f"Lease fence is stale for resource {claim.resource_key!r}" + ) + + +def _lease_claim(lease: DistributedLease) -> LeaseClaim: + if lease.holder_node_id is None or lease.holder_incarnation is None: + raise LeaseUnavailable("Distributed lease has no active holder") + return LeaseClaim( + installation_id=lease.installation_id, + resource_key=lease.resource_key, + holder_node_id=lease.holder_node_id, + holder_incarnation=lease.holder_incarnation, + fencing_token=int(lease.fencing_token), + expires_at=_as_utc(lease.expires_at), + ) + + +def _as_utc(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def _iso(value: datetime | None) -> str | None: + return _as_utc(value).isoformat() if value is not None else None + + +__all__ = [ + "DistributedLease", + "LeaseClaim", + "LeaseUnavailable", + "RuntimeCoordinationError", + "RuntimeIdentity", + "RuntimeNode", + "RuntimeNodeState", + "StaleFence", + "acquire_lease", + "assert_lease_fence", + "cancel_runtime_node_drain", + "heartbeat_runtime_node", + "list_runtime_nodes", + "register_runtime_node", + "release_lease", + "renew_lease", + "request_runtime_node_drain", + "runtime_identity", + "runtime_node_is_draining", + "stop_runtime_node", +] diff --git a/src/govoplan_core/db/bootstrap.py b/src/govoplan_core/db/bootstrap.py index 9be4e7f..9959517 100644 --- a/src/govoplan_core/db/bootstrap.py +++ b/src/govoplan_core/db/bootstrap.py @@ -32,6 +32,8 @@ def create_all_tables() -> None: # model metadata with the shared SQLAlchemy base before create_all runs. from govoplan_core.admin import models as core_admin_models # noqa: F401 from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 + from govoplan_core.core import recovery as core_recovery_models # noqa: F401 + from govoplan_core.core import runtime_coordination as core_runtime_models # noqa: F401 from govoplan_core.security import credential_envelopes as core_credential_models # noqa: F401 raw_enabled_modules = load_startup_enabled_modules(settings.enabled_modules) diff --git a/src/govoplan_core/db/migration_lock.py b/src/govoplan_core/db/migration_lock.py new file mode 100644 index 0000000..c1825af --- /dev/null +++ b/src/govoplan_core/db/migration_lock.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from contextlib import contextmanager +import hashlib +import time +from collections.abc import Iterator + +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url +from sqlalchemy.exc import SQLAlchemyError + + +class MigrationLockTimeout(RuntimeError): + pass + + +def migration_lock_key(installation_id: str, migration_track: str) -> int: + payload = f"govoplan\0{installation_id}\0{migration_track}".encode("utf-8") + return int.from_bytes(hashlib.sha256(payload).digest()[:8], "big", signed=True) + + +@contextmanager +def deployment_migration_lock( + database_url: str, + *, + installation_id: str, + migration_track: str, + timeout_seconds: float = 900.0, + poll_seconds: float = 2.0, +) -> Iterator[None]: + """Serialize all release migrations before coordination tables are available.""" + + if timeout_seconds < 0 or poll_seconds <= 0: + raise ValueError("Migration lock timeout must be non-negative and polling positive") + if make_url(database_url).get_backend_name() != "postgresql": + yield + return + key = migration_lock_key(installation_id, migration_track) + engine = create_engine(database_url) + connection = engine.connect() + acquired = False + try: + deadline = time.monotonic() + timeout_seconds + while True: + acquired = bool( + connection.execute( + text("SELECT pg_try_advisory_lock(:key)"), + {"key": key}, + ).scalar_one() + ) + if acquired: + break + if time.monotonic() >= deadline: + raise MigrationLockTimeout( + "Timed out waiting for the deployment-wide database migration lock" + ) + time.sleep(min(poll_seconds, max(0.05, deadline - time.monotonic()))) + yield + finally: + if acquired: + try: + connection.execute( + text("SELECT pg_advisory_unlock(:key)"), + {"key": key}, + ) + except SQLAlchemyError: + # Closing the physical connection releases session locks. + pass + connection.close() + engine.dispose() + + +__all__ = [ + "MigrationLockTimeout", + "deployment_migration_lock", + "migration_lock_key", +] diff --git a/src/govoplan_core/db/migrations.py b/src/govoplan_core/db/migrations.py index 70d7904..14b869a 100644 --- a/src/govoplan_core/db/migrations.py +++ b/src/govoplan_core/db/migrations.py @@ -18,6 +18,8 @@ from sqlalchemy import create_engine, inspect, text from govoplan_core.core.migrations import MigrationMetadataPlan, migration_metadata_plan from govoplan_core.core import change_sequence as core_change_sequence_models # noqa: F401 - populate core metadata +from govoplan_core.core import recovery as core_recovery_models # noqa: F401 - populate core metadata +from govoplan_core.core import runtime_coordination as core_runtime_models # noqa: F401 - populate core metadata from govoplan_core.security import credential_envelopes as core_credential_models # noqa: F401 - populate core metadata from govoplan_core.core.change_sequence import ChangeSequenceEntry, ChangeSequenceRetentionFloor from govoplan_core.core.module_management import load_startup_enabled_modules, startup_candidate_module_ids @@ -590,6 +592,51 @@ def database_revision(database_url: str | None = None) -> str | None: engine.dispose() +def database_migration_heads( + database_url: str | None = None, +) -> tuple[str, ...]: + url = database_url or settings.database_url + engine = create_engine(url) + try: + with engine.connect() as connection: + return tuple( + sorted(MigrationContext.configure(connection).get_current_heads()) + ) + finally: + engine.dispose() + + +def configured_migration_heads( + database_url: str | None = None, + *, + enabled_modules: tuple[str, ...] | list[str] | None = None, + manifest_factories: tuple[ManifestFactory, ...] = (), + migration_track: str | None = None, +) -> tuple[str, ...]: + config = alembic_config( + database_url=database_url, + enabled_modules=enabled_modules, + manifest_factories=manifest_factories, + migration_track=migration_track, + ) + return tuple(sorted(ScriptDirectory.from_config(config).get_heads())) + + +def database_is_at_configured_heads( + database_url: str | None = None, + *, + enabled_modules: tuple[str, ...] | list[str] | None = None, + manifest_factories: tuple[ManifestFactory, ...] = (), + migration_track: str | None = None, +) -> bool: + return database_migration_heads(database_url) == configured_migration_heads( + database_url, + enabled_modules=enabled_modules, + manifest_factories=manifest_factories, + migration_track=migration_track, + ) + + def _has_columns(inspector, table_name: str, required: set[str]) -> bool: try: actual = {column["name"] for column in inspector.get_columns(table_name)} diff --git a/src/govoplan_core/server/default_config.py b/src/govoplan_core/server/default_config.py index d89feda..e7a4477 100644 --- a/src/govoplan_core/server/default_config.py +++ b/src/govoplan_core/server/default_config.py @@ -3,6 +3,7 @@ from __future__ import annotations from contextlib import asynccontextmanager from fastapi import Depends, FastAPI +from fastapi import HTTPException, status from sqlalchemy.engine import make_url from govoplan_core.auth import ApiPrincipal, require_scope @@ -10,6 +11,7 @@ from govoplan_core.core.registry import PlatformRegistry from govoplan_core.db.bootstrap import bootstrap_dev_data, create_all_tables from govoplan_core.db.session import get_database from govoplan_core.server.config import GovoplanServerConfig +from govoplan_core.server.runtime_agent import RuntimeNodeAgent from govoplan_core.settings import Settings, settings @@ -41,15 +43,36 @@ async def lifespan(app: FastAPI): "reconcile_workflow_definitions", ): lifecycle.reconcile_workflow_definitions() - yield + registry = getattr(app.state, "govoplan_registry", None) + module_ids = ( + tuple(manifest.id for manifest in registry.manifests()) + if registry is not None + else () + ) + runtime_agent = RuntimeNodeAgent( + settings=settings, + software_version=app.version, + module_ids=module_ids, + metadata={"process": "api"}, + ) + await runtime_agent.start() + app.state.govoplan_runtime_agent = runtime_agent + try: + yield + finally: + await runtime_agent.stop() def _cors_origins(value: str) -> list[str]: return [item.strip() for item in value.split(",") if item.strip()] -def register_health_details(app: FastAPI, registry: PlatformRegistry, config_settings: object | None) -> None: - active_settings = config_settings if isinstance(config_settings, Settings) else settings +def register_health_details( + app: FastAPI, registry: PlatformRegistry, config_settings: object | None +) -> None: + active_settings = ( + config_settings if isinstance(config_settings, Settings) else settings + ) @app.get("/health/details") def health_details( @@ -63,13 +86,39 @@ def register_health_details(app: FastAPI, registry: PlatformRegistry, config_set "modules": [manifest.id for manifest in registry.manifests()], "storage": { "backend": active_settings.file_storage_backend, - "local_root": active_settings.file_storage_local_root if active_settings.file_storage_backend == "local" else None, - "endpoint": active_settings.file_storage_s3_endpoint_url or active_settings.s3_endpoint_url, - "bucket": active_settings.file_storage_s3_bucket or active_settings.s3_bucket, - "region": active_settings.file_storage_s3_region or active_settings.s3_region, + "local_root": active_settings.file_storage_local_root + if active_settings.file_storage_backend == "local" + else None, + "endpoint": active_settings.file_storage_s3_endpoint_url + or active_settings.s3_endpoint_url, + "bucket": active_settings.file_storage_s3_bucket + or active_settings.s3_bucket, + "region": active_settings.file_storage_s3_region + or active_settings.s3_region, }, } + @app.get("/health/ready") + def health_ready(): + runtime_agent = getattr(app.state, "govoplan_runtime_agent", None) + if runtime_agent is not None and not runtime_agent.coordination_healthy: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={ + "status": "coordination_unavailable", + "node_id": runtime_agent.identity.node_id, + }, + ) + if runtime_agent is not None and runtime_agent.draining: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={ + "status": "draining", + "node_id": runtime_agent.identity.node_id, + }, + ) + return {"status": "ready"} + def get_server_config() -> GovoplanServerConfig: return GovoplanServerConfig( diff --git a/src/govoplan_core/server/platform.py b/src/govoplan_core/server/platform.py index fdb5b0f..01a7b6f 100644 --- a/src/govoplan_core/server/platform.py +++ b/src/govoplan_core/server/platform.py @@ -168,6 +168,15 @@ def create_platform_router(settings: object | None = None) -> APIRouter: "dependencies": list(manifest.dependencies), "optional_dependencies": list(manifest.optional_dependencies), "enabled": True, + "architecture": ( + manifest.architecture.to_dict() + if manifest.architecture is not None + else None + ), + "external_providers": [ + declaration.to_dict() + for declaration in manifest.external_providers + ], "runtime_ui_capabilities": _runtime_ui_capabilities(manifest.id, settings, registry), "nav": [_nav_item_payload(item, manifest.id) for item in manifest.nav_items], "frontend": _frontend_payload(manifest), diff --git a/src/govoplan_core/server/runtime_agent.py b/src/govoplan_core/server/runtime_agent.py new file mode 100644 index 0000000..61c862c --- /dev/null +++ b/src/govoplan_core/server/runtime_agent.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from govoplan_core.core.runtime_coordination import ( + RuntimeIdentity, + heartbeat_runtime_node, + register_runtime_node, + runtime_identity, + stop_runtime_node, +) +from govoplan_core.db.session import get_database + + +logger = logging.getLogger("govoplan.runtime") + + +class RuntimeNodeAgent: + """Register one process in the shared runtime directory and heartbeat it.""" + + def __init__( + self, + *, + settings: object, + software_version: str, + module_ids: tuple[str, ...], + role: str | None = None, + node_id: str | None = None, + queues: tuple[str, ...] | None = None, + metadata: dict[str, Any] | None = None, + ) -> None: + self.settings = settings + self.identity: RuntimeIdentity = runtime_identity( + settings, + software_version=software_version, + module_ids=module_ids, + role=role, + node_id=node_id, + queues=queues, + ) + self.metadata = dict(metadata or {}) + self.draining = False + self.coordination_healthy = False + self._task: asyncio.Task[None] | None = None + self._stopping = False + + @property + def heartbeat_seconds(self) -> int: + return max( + 2, + int(getattr(self.settings, "runtime_heartbeat_seconds", 15)), + ) + + async def start(self) -> None: + await asyncio.to_thread(self._register) + self._task = asyncio.create_task( + self._heartbeat_loop(), + name=f"govoplan-runtime-heartbeat:{self.identity.node_id}", + ) + + async def stop(self) -> None: + self._stopping = True + task = self._task + self._task = None + if task is not None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + try: + await asyncio.to_thread(self._mark_stopped) + except Exception: # noqa: BLE001 - shutdown must continue + logger.exception( + "runtime node stop marker failed node_id=%s", + self.identity.node_id, + ) + + def _register(self) -> None: + with get_database().SessionLocal() as session: + node = register_runtime_node( + session, + self.identity, + metadata=self.metadata, + ) + session.commit() + self.draining = node.state == "draining" + self.coordination_healthy = True + + def _heartbeat(self) -> None: + with get_database().SessionLocal() as session: + node = heartbeat_runtime_node( + session, + self.identity, + metadata=self.metadata, + ) + session.commit() + self.draining = node.state == "draining" + self.coordination_healthy = True + + def _mark_stopped(self) -> None: + with get_database().SessionLocal() as session: + stop_runtime_node(session, self.identity) + session.commit() + + async def _heartbeat_loop(self) -> None: + while not self._stopping: + await asyncio.sleep(self.heartbeat_seconds) + await self._heartbeat_once() + + async def _heartbeat_once(self) -> bool: + try: + await asyncio.to_thread(self._heartbeat) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - a later heartbeat can recover + self.coordination_healthy = False + logger.exception( + "runtime heartbeat failed node_id=%s", + self.identity.node_id, + ) + return False + self.coordination_healthy = True + return True + + +__all__ = ["RuntimeNodeAgent"] diff --git a/src/govoplan_core/settings.py b/src/govoplan_core/settings.py index d502b1f..211dfcd 100644 --- a/src/govoplan_core/settings.py +++ b/src/govoplan_core/settings.py @@ -6,6 +6,40 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=None, extra="ignore") app_env: str = Field(default="dev", alias="APP_ENV") + installation_id: str = Field( + default="govoplan-local", + alias="GOVOPLAN_INSTALLATION_ID", + ) + state_profile: str = Field( + default="local", + alias="GOVOPLAN_STATE_PROFILE", + ) + runtime_role: str = Field(default="api", alias="GOVOPLAN_RUNTIME_ROLE") + runtime_node_id: str | None = Field(default=None, alias="GOVOPLAN_NODE_ID") + runtime_heartbeat_seconds: int = Field( + default=15, + ge=2, + le=300, + alias="GOVOPLAN_RUNTIME_HEARTBEAT_SECONDS", + ) + runtime_stale_after_seconds: int = Field( + default=60, + ge=10, + le=3600, + alias="GOVOPLAN_RUNTIME_STALE_AFTER_SECONDS", + ) + runtime_expected_api_replicas: int = Field( + default=1, + ge=1, + le=1000, + alias="GOVOPLAN_EXPECTED_API_REPLICAS", + ) + runtime_expected_worker_replicas: int = Field( + default=0, + ge=0, + le=1000, + alias="GOVOPLAN_EXPECTED_WORKER_REPLICAS", + ) module_live_apply_enabled: bool | None = Field( default=None, alias="GOVOPLAN_MODULE_LIVE_APPLY_ENABLED", @@ -78,6 +112,10 @@ class Settings(BaseSettings): default=False, alias="FILE_STORAGE_S3_DEPLOYMENT_MANAGED", ) + file_storage_s3_endpoint_trusted: bool = Field( + default=False, + alias="FILE_STORAGE_S3_ENDPOINT_TRUSTED", + ) file_upload_max_bytes: int = Field(default=50 * 1024 * 1024, alias="FILE_UPLOAD_MAX_BYTES") file_upload_zip_max_bytes: int = Field(default=250 * 1024 * 1024, alias="FILE_UPLOAD_ZIP_MAX_BYTES") file_archive_max_entries: int = Field(default=10_000, ge=1, alias="FILE_ARCHIVE_MAX_ENTRIES") diff --git a/tests/test_configuration_package_architecture.py b/tests/test_configuration_package_architecture.py new file mode 100644 index 0000000..9e9259b --- /dev/null +++ b/tests/test_configuration_package_architecture.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import unittest + +from govoplan_core.core.configuration_packages import ( + ConfigurationModuleRequirement, + ConfigurationPackageEvidence, + ConfigurationPackageManifest, + ConfigurationPackageParent, + ConfigurationPreflightContext, + ConfigurationProviderExpectation, + dry_run_configuration_package, + validate_configuration_package_derivation, +) + + +def _evidence(*kinds: str) -> tuple[ConfigurationPackageEvidence, ...]: + return tuple( + ConfigurationPackageEvidence( + kind=kind, # type: ignore[arg-type] + reference=f"evidence/{kind}.json", + summary=f"Evidence for {kind}.", + checksum=f"sha256:{'a' * 64}", + ) + for kind in kinds + ) + + +class ConfigurationPackageArchitectureTests(unittest.TestCase): + def test_legacy_package_defaults_to_product_and_round_trips(self) -> None: + package = ConfigurationPackageManifest.from_mapping( + {"package_id": "example", "name": "Example", "version": "1.0.0"} + ) + + self.assertEqual(package.package_class, "product") + self.assertEqual( + ConfigurationPackageManifest.from_mapping(package.to_dict()), + package, + ) + + def test_reference_package_requires_operational_evidence(self) -> None: + with self.assertRaisesRegex(ValueError, "accessibility"): + ConfigurationPackageManifest( + package_id="reference.example", + name="Example reference", + version="1.0.0", + package_class="reference", + evidence=_evidence("target_test", "recovery"), + ) + + def test_reference_evidence_requires_content_checksums(self) -> None: + with self.assertRaisesRegex(ValueError, "without checksums"): + ConfigurationPackageManifest( + package_id="reference.unbound", + name="Unbound reference", + version="1.0.0", + package_class="reference", + evidence=tuple( + ConfigurationPackageEvidence( + kind=kind, # type: ignore[arg-type] + reference=f"evidence/{kind}.json", + summary=f"{kind} evidence.", + checksum=( + None + if kind == "target_test" + else f"sha256:{'b' * 64}" + ), + ) + for kind in ( + "target_test", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "documentation", + ) + ), + ) + + def test_integration_package_preflight_checks_provider_contract_and_health(self) -> None: + package = ConfigurationPackageManifest( + package_id="integration.example", + name="Example integration", + version="1.0.0", + package_class="integration", + evidence=_evidence( + "target_test", "recovery", "operations", "documentation" + ), + provider_expectations=( + ConfigurationProviderExpectation( + provider_id="connectors.example", + authority_mode="external_mirror", + minimum_maturity="read", + ), + ), + ) + + healthy = dry_run_configuration_package( + package, + (), + ConfigurationPreflightContext( + external_provider_declarations={ + "connectors.example": { + "authority_modes": ["external_mirror"], + "maturity": "synchronize", + } + }, + external_provider_states={ + "connectors.example": {"health": "healthy"} + }, + ), + ) + unhealthy = dry_run_configuration_package( + package, + (), + ConfigurationPreflightContext( + external_provider_declarations={ + "connectors.example": { + "authority_modes": ["external_authoritative"], + "maturity": "discover", + } + }, + external_provider_states={ + "connectors.example": {"health": "error"} + }, + ), + ) + + self.assertFalse( + any(item.severity == "blocker" for item in healthy.diagnostics) + ) + self.assertEqual( + { + "external_provider_authority_mode_unsupported", + "external_provider_maturity_insufficient", + "external_provider_unhealthy", + }, + {item.code for item in unhealthy.diagnostics}, + ) + + def test_provider_preflight_selects_the_requested_runtime_binding(self) -> None: + package = ConfigurationPackageManifest( + package_id="integration.calendar", + name="Calendar integration", + version="1.0.0", + package_class="integration", + evidence=_evidence( + "target_test", "recovery", "operations", "documentation" + ), + provider_expectations=( + ConfigurationProviderExpectation( + provider_id="calendar.caldav_sync", + authority_mode="governed_sync", + minimum_maturity="synchronize", + binding_ref="calendar:sync-source:required", + freshness_expectation="current", + recovery_expectation="ready", + ), + ), + ) + context = ConfigurationPreflightContext( + external_provider_declarations={ + "calendar.caldav_sync": { + "authority_modes": ["governed_sync"], + "maturity": "synchronize", + "behavior": {}, + } + }, + external_provider_states={ + "calendar.caldav_sync": { + "health": "warning", + "bindings": [ + { + "binding_ref": "calendar:sync-source:other", + "authority_mode": "governed_sync", + "health": "error", + "freshness": "stale", + "recovery": "attention", + }, + { + "binding_ref": "calendar:sync-source:required", + "authority_mode": "governed_sync", + "health": "healthy", + "freshness": "current", + "recovery": "ready", + }, + ], + } + }, + ) + + result = dry_run_configuration_package(package, (), context) + + self.assertFalse( + any(item.severity == "blocker" for item in result.diagnostics) + ) + + def test_derived_package_may_tighten_but_not_remove_parent_constraints(self) -> None: + parent = ConfigurationPackageManifest( + package_id="product.base", + name="Base", + version="1.0.0", + required_modules=( + ConfigurationModuleRequirement("connectors", "1.0.0"), + ), + required_capabilities=("connectors.profiles",), + provider_expectations=( + ConfigurationProviderExpectation( + provider_id="connectors.example", + authority_mode="external_mirror", + minimum_maturity="read", + ), + ), + ) + child = ConfigurationPackageManifest( + package_id="sector.example", + name="Sector example", + version="1.0.0", + package_class="sector", + parents=( + ConfigurationPackageParent("product.base", "1.0.0"), + ), + evidence=_evidence("documentation"), + ) + + codes = { + item.code + for item in validate_configuration_package_derivation(child, parent) + } + self.assertEqual( + { + "package_parent_module_constraint_loosened", + "package_parent_capability_constraint_loosened", + "package_parent_provider_constraint_removed", + }, + codes, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_external_reference_contract.py b/tests/test_external_reference_contract.py index 47d917f..780f9c1 100644 --- a/tests/test_external_reference_contract.py +++ b/tests/test_external_reference_contract.py @@ -15,12 +15,14 @@ class ExternalReferenceContractTests(unittest.TestCase): object_type="work_package", object_id="42", maturity="synchronize", + authority_mode="governed_sync", canonical_url="https://projects.example.test/work_packages/42", ) self.assertEqual("openproject-main:work_package:42", reference.identity_key) self.assertTrue(reference.supports("read")) self.assertFalse(reference.supports("replace")) + self.assertEqual("governed_sync", reference.to_dict()["authority_mode"]) def test_reference_rejects_credentials_in_urls(self) -> None: with self.assertRaises(ExternalReferenceValidationError): @@ -31,6 +33,19 @@ class ExternalReferenceContractTests(unittest.TestCase): canonical_url="https://user:secret@example.test/wiki/Main_Page", ) + def test_authority_mode_must_match_maturity(self) -> None: + with self.assertRaisesRegex( + ExternalReferenceValidationError, + "Governed-sync references require synchronize maturity", + ): + ExternalObjectReference( + system="openproject", + object_type="work_package", + object_id="7", + maturity="read", + authority_mode="governed_sync", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_fenced_run.py b/tests/test_fenced_run.py new file mode 100644 index 0000000..e64ab72 --- /dev/null +++ b/tests/test_fenced_run.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import subprocess + +from govoplan_core.commands import fenced_run +from govoplan_core.core.runtime_coordination import LeaseClaim + + +class _IgnoringProcess: + def __init__(self) -> None: + self.terminated = False + self.killed = False + self.wait_count = 0 + + def poll(self): + return None if not self.killed else -9 + + def terminate(self) -> None: + self.terminated = True + + def kill(self) -> None: + self.killed = True + + def wait(self, *, timeout: float): + self.wait_count += 1 + if not self.killed: + raise subprocess.TimeoutExpired("command", timeout) + return -9 + + +def test_fenced_process_is_killed_when_it_ignores_termination() -> None: + process = _IgnoringProcess() + + fenced_run._terminate_process(process, timeout_seconds=0.01) + + assert process.terminated is True + assert process.killed is True + assert process.wait_count == 2 + + +def test_lease_release_is_best_effort_after_database_loss(monkeypatch) -> None: + claim = LeaseClaim( + installation_id="installation-1", + resource_key="scheduler", + holder_node_id="scheduler-1", + holder_incarnation="incarnation-1", + fencing_token=1, + expires_at=datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc), + ) + + monkeypatch.setattr( + fenced_run, + "get_database", + lambda: (_ for _ in ()).throw(RuntimeError("database unavailable")), + ) + + fenced_run._release(claim) diff --git a/tests/test_install_config.py b/tests/test_install_config.py index 33e0b38..6f9ebfb 100644 --- a/tests/test_install_config.py +++ b/tests/test_install_config.py @@ -186,6 +186,44 @@ class InstallConfigTests(unittest.TestCase): ).file_storage_s3_deployment_managed ) + def test_external_s3_requires_explicit_https_deployment_trust(self) -> None: + base = { + "APP_ENV": "development", + "FILE_STORAGE_BACKEND": "s3", + "FILE_STORAGE_S3_ENDPOINT_URL": "https://s3.example.test", + "FILE_STORAGE_S3_REGION": "eu-test-1", + "FILE_STORAGE_S3_ACCESS_KEY_ID": "access", + "FILE_STORAGE_S3_SECRET_ACCESS_KEY": "secret", + "FILE_STORAGE_S3_BUCKET": "files", + "FILE_STORAGE_S3_DEPLOYMENT_MANAGED": "false", + } + rejected = validate_runtime_configuration(base, profile="development") + accepted = validate_runtime_configuration( + {**base, "FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true"}, + profile="development", + ) + insecure = validate_runtime_configuration( + { + **base, + "FILE_STORAGE_S3_ENDPOINT_URL": "http://s3.example.test", + "FILE_STORAGE_S3_ENDPOINT_TRUSTED": "true", + }, + profile="development", + ) + + self.assertIn( + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + {issue.key for issue in rejected.errors}, + ) + self.assertNotIn( + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + {issue.key for issue in accepted.errors}, + ) + self.assertIn( + "FILE_STORAGE_S3_ENDPOINT_URL", + {issue.key for issue in insecure.errors}, + ) + def test_connector_deployment_allowlists_require_exact_names_and_absolute_paths(self) -> None: result = validate_runtime_configuration( { @@ -215,6 +253,7 @@ class InstallConfigTests(unittest.TestCase): self.assertIn("GOVOPLAN_HTTP_HSTS_SECONDS=31536000", self_hosted) self.assertIn("AUTH_LOGIN_THROTTLE_IDENTITY_LIMIT=10", self_hosted) self.assertIn("FILE_STORAGE_S3_DEPLOYMENT_MANAGED=false", self_hosted) + self.assertIn("FILE_STORAGE_S3_ENDPOINT_TRUSTED=false", self_hosted) self.assertIn("GOVOPLAN_INSTALL_PROFILE=production-like", production_like) self.assertNotIn("MASTER_KEY_B64= InstitutionalReference: + owners = { + "institution": "tenancy", + "organization_unit": "organizations", + "function": "organizations", + "function_assignment": "idm", + "mandate": "cases", + "jurisdiction": "cases", + "service": "portal", + "form": "forms", + "form_submission": "forms_runtime", + "case": "cases", + "party": "cases", + "decision": "committee", + "work_item": "workflow_engine", + "workflow": "workflow_engine", + "approval": "workflow_engine", + "record": "audit", + } + return InstitutionalReference( + kind=kind, + owner_module=owners[kind], + object_id=object_id, + tenant_id=tenant_id, + version="3", + valid_at=NOW, + label=f"Protected label for {object_id}", + ) + + +def _context() -> GovernedContextEnvelope: + external = ExternalObjectReference( + system="gis.example", + object_type="administrative_area", + object_id="area-7", + maturity="read", + authority_mode="external_mirror", + canonical_url="https://gis.example/areas/7", + observed_at=NOW, + ) + return GovernedContextEnvelope( + tenant_id="tenant-1", + temporal=TemporalRevision( + revision="7", + valid_from=NOW - timedelta(days=1), + valid_to=NOW + timedelta(days=1), + recorded_at=NOW, + change_reason="Case state changed.", + ), + actor=ActorRepresentationReference( + tenant_id="tenant-1", + account_id="account-1", + identity_id="identity-1", + represented_function_ref=_reference("function", "function-1"), + function_assignment_ref=_reference( + "function_assignment", "assignment-1" + ), + mandate_ref=_reference("mandate", "mandate-1"), + ), + institution_ref=_reference("institution", "institution-1"), + organization_unit_ref=_reference("organization_unit", "unit-1"), + function_ref=_reference("function", "function-1"), + mandate_ref=_reference("mandate", "mandate-1"), + jurisdiction_refs=(_reference("jurisdiction", "jurisdiction-1"),), + service_ref=_reference("service", "service-1"), + case_ref=_reference("case", "case-1"), + party_refs=(_reference("party", "party-1"),), + work_item_ref=_reference("work_item", "work-item-1"), + workflow_ref=_reference("workflow", "workflow-1"), + approval_refs=(_reference("approval", "approval-1"),), + decision_ref=_reference("decision", "decision-1"), + record_refs=(_reference("record", "record-1"),), + legal_bases=( + LegalBasisReference( + kind="law", + authority="Example legislature", + reference="law-1/section-3", + version="2026-01", + effective_from=NOW - timedelta(days=30), + title="Protected legal title", + inspection_url="https://law.example/section-3", + ), + ), + evidence=( + EvidenceReference( + kind="document", + owner_module="files", + evidence_id="file-1", + tenant_id="tenant-1", + version="4", + checksum="sha256:abcd", + captured_at=NOW, + inspection_url="https://govoplan.example/files/file-1", + ), + ), + information_governance=InformationGovernanceReference( + classification="confidential", + purposes=("case decision",), + legal_basis_refs=("law-1/section-3",), + retention_policy_ref="records:case-default", + disclosure_state="restricted", + ), + external_sources=( + ExternalSourceReference( + reference=external, + authority_mode="external_mirror", + maturity="read", + freshness_state="current", + health_state="ok", + ), + ), + ) + + +class InstitutionalContractTests(unittest.TestCase): + def test_context_round_trip_preserves_versioned_references(self) -> None: + context = _context() + + restored = GovernedContextEnvelope.from_mapping( + context.to_dict(include_inspection=True) + ) + + self.assertEqual(context.tenant_id, restored.tenant_id) + self.assertEqual("mandate-1", restored.mandate_ref.object_id) + self.assertEqual( + "external_mirror", restored.external_sources[0].authority_mode + ) + self.assertEqual("workflow-1", restored.workflow_ref.object_id) + self.assertEqual("approval-1", restored.approval_refs[0].object_id) + self.assertEqual("record-1", restored.record_refs[0].object_id) + self.assertTrue(restored.temporal.effective_at(NOW)) + + def test_cross_tenant_reference_is_rejected(self) -> None: + with self.assertRaisesRegex(InstitutionalContextError, "different tenant"): + GovernedContextEnvelope( + tenant_id="tenant-1", + temporal=TemporalRevision(revision="1", recorded_at=NOW), + case_ref=_reference("case", "case-2", tenant_id="tenant-2"), + ) + + def test_default_serialization_redacts_labels_and_inspection_links(self) -> None: + payload = _context().to_dict() + + self.assertIsNone(payload["case_ref"]["label"]) + self.assertIsNone(payload["legal_bases"][0]["title"]) + self.assertIsNone(payload["legal_bases"][0]["inspection_url"]) + self.assertIsNone(payload["evidence"][0]["inspection_url"]) + + def test_event_and_action_envelopes_carry_reference_context(self) -> None: + context = _context() + event = PlatformEvent( + type="committee.decision.recorded", + module_id="committee", + institutional_context=context, + ) + action = ActionExecutionRequest( + tenant_id="tenant-1", + action_key="committee.decision.publish", + input={"decision_ref": "decision-1"}, + idempotency_key="decision-1:publish:1", + invocation=AutomationInvocation(kind="workflow"), + institutional_context=context, + ) + + self.assertEqual( + "decision-1", + event.to_dict()["institutional_context"]["decision_ref"]["object_id"], + ) + self.assertIs(context, action.institutional_context) + + def test_formal_decision_redacts_reasoning_but_keeps_evidence_refs(self) -> None: + context = _context() + decision = FormalDecision( + reference=_reference("decision", "decision-1"), + temporal=context.temporal, + decision_type="permit", + subject_refs=(_reference("case", "case-1"),), + state="decided", + authority_context=context, + fact_evidence=context.evidence, + legal_bases=context.legal_bases, + operative_result="Permit granted.", + reasoning="Protected reasoning.", + ) + + redacted = decision.to_dict() + protected = decision.to_dict(include_protected=True) + + self.assertIsNone(redacted["reasoning"]) + self.assertEqual("Protected reasoning.", protected["reasoning"]) + self.assertEqual( + "file-1", redacted["fact_evidence"][0]["evidence_id"] + ) + + restored = FormalDecision.from_mapping(protected) + self.assertEqual("Protected reasoning.", restored.reasoning) + self.assertEqual("decision-1", restored.reference.object_id) + + def test_semantic_provider_dtos_round_trip_without_domain_tables(self) -> None: + context = _context() + mandate = MandateDefinition( + reference=_reference("mandate", "mandate-1"), + temporal=context.temporal, + task_types=("committee.formal_decision",), + authority_types=("permit",), + organization_unit_refs=( + _reference("organization_unit", "unit-1"), + ), + function_refs=(_reference("function", "function-1"),), + jurisdiction_refs=context.jurisdiction_refs, + legal_bases=context.legal_bases, + evidence=context.evidence, + ) + resolution = MandateResolution( + competent=True, + mandates=(mandate,), + explanation="Authority matched.", + evidence=context.evidence, + ) + service = ServiceDefinition( + reference=_reference("service", "service-1"), + key="permit.service", + temporal=context.temporal, + title="Permit service", + audience=("residents",), + legal_bases=context.legal_bases, + channels=("portal", "mail"), + responsible_organization_ref=_reference( + "organization_unit", "unit-1" + ), + responsible_function_ref=_reference("function", "function-1"), + mandate_ref=_reference("mandate", "mandate-1"), + jurisdiction_refs=context.jurisdiction_refs, + bindings=(ServiceBinding("workflow", "workflow:permit"),), + publication_state="published", + ) + represented = _reference("party", "party-applicant") + representative = _reference("party", "party-agent") + party = ProcedureParty( + reference=represented, + procedure_ref=_reference("case", "case-1"), + role="applicant", + subject=PartySubjectReference( + kind="identity", + provider="identity", + subject_id="identity-1", + tenant_id="tenant-1", + version="2", + ), + temporal=context.temporal, + preferred_channels=("portal",), + permitted_channels=("portal", "mail"), + delivery_recipient=True, + representations=( + PartyRepresentation( + representative_party_ref=representative, + represented_party_ref=represented, + power_ref="power-1", + permitted_actions=("submit", "receive"), + temporal=context.temporal, + evidence=context.evidence, + ), + ), + contact_snapshot_refs=("addresses:snapshot-1",), + evidence=context.evidence, + ) + + restored_resolution = MandateResolution.from_mapping( + resolution.to_dict(include_inspection=True) + ) + restored_service = ServiceDefinition.from_mapping(service.to_dict()) + restored_party = ProcedureParty.from_mapping( + party.to_dict(include_inspection=True) + ) + + self.assertTrue(restored_resolution.competent) + self.assertEqual( + "committee.formal_decision", + restored_resolution.mandates[0].task_types[0], + ) + self.assertEqual("workflow:permit", restored_service.bindings[0].reference) + self.assertEqual("identity-1", restored_party.subject.subject_id) + self.assertEqual( + "party-agent", + restored_party.representations[0].representative_party_ref.object_id, + ) + + def test_invalid_semantic_states_are_rejected(self) -> None: + with self.assertRaisesRegex(InstitutionalContextError, "disclosure state"): + InformationGovernanceReference( + classification="internal", + purposes=("test",), + disclosure_state="invalid", # type: ignore[arg-type] + ) + + def test_service_launch_contract_requires_exact_same_tenant_targets(self) -> None: + service_ref = _reference("service", "service-1") + binding = ServiceBinding("case", "permit") + request = ServiceLaunchRequest( + service_ref=service_ref, + binding=binding, + idempotency_key="launch-1", + requested_at=NOW, + parameters={"title": "Permit application"}, + ) + result = ServiceLaunchResult( + service_ref=service_ref, + binding=binding, + state="started", + target_ref=_reference("case", "case-1"), + href="/cases/case-1", + metadata={"case_number": "PERMIT-1"}, + ) + + self.assertEqual("launch-1", request.idempotency_key) + self.assertEqual("case-1", result.to_dict()["target_ref"]["object_id"]) + with self.assertRaisesRegex(InstitutionalContextError, "another tenant"): + ServiceLaunchResult( + service_ref=service_ref, + binding=binding, + state="started", + target_ref=_reference("case", "case-2", tenant_id="tenant-2"), + metadata={}, + ) + + def test_form_definition_contract_round_trips_exact_schema(self) -> None: + definition = FormDefinition( + reference=_reference("form", "permit-application"), + key="permit-application", + temporal=TemporalRevision( + revision="3", + recorded_at=NOW, + change_reason="Add delivery preference.", + ), + title="Permit application", + description="Apply for a permit.", + fields=( + FormFieldDefinition( + key="email", + label="Email address", + value_type="email", + required=True, + constraints={"max_length": 255}, + ), + FormFieldDefinition( + key="delivery", + label="Delivery", + value_type="choice", + options=("portal", "mail"), + ), + ), + max_attachments=3, + signature_requirement="optional", + policy_refs=("policy:permit-intake",), + handoff_kinds=("case",), + ) + + restored = FormDefinition.from_mapping(definition.to_dict()) + + self.assertEqual("3", restored.reference.version) + self.assertEqual("email", restored.fields[0].key) + self.assertEqual(("portal", "mail"), restored.fields[1].options) + with self.assertRaisesRegex(InstitutionalContextError, "unique"): + replace(restored, fields=(restored.fields[0], restored.fields[0])) + + def test_service_launch_redirect_rejects_credentialed_urls(self) -> None: + with self.assertRaisesRegex(InstitutionalContextError, "credential-free"): + ServiceLaunchResult( + service_ref=_reference("service", "service-1"), + binding=ServiceBinding("external", "external-entry"), + state="redirect", + href="https://user:secret@example.test/start", + metadata={}, + ) + + with self.assertRaisesRegex(InstitutionalContextError, "must be a boolean"): + ServiceBinding.from_mapping( + { + "kind": "workflow", + "reference": "workflow:permit", + "required": "false", + } + ) + + def test_service_derivation_can_only_tighten_parent_constraints(self) -> None: + parent = ServiceDefinition( + reference=_reference("service", "permit-template"), + key="permit.apply", + temporal=TemporalRevision( + revision="7", + valid_from=NOW - timedelta(days=1), + valid_to=NOW + timedelta(days=10), + recorded_at=NOW - timedelta(days=2), + ), + title="Permit service", + audience=("resident", "business"), + prerequisites=("registered-address",), + required_evidence_types=("application",), + channels=("portal", "mail"), + bindings=(ServiceBinding("case", "permit-case"),), + availability_requirements=( + ServiceAvailabilityRequirement("policy", "permit-access"), + ), + publication_state="published", + ) + temporal = TemporalRevision( + revision="8", + valid_from=NOW, + valid_to=NOW + timedelta(days=5), + recorded_at=NOW, + change_reason="Tenant specialization.", + ) + reference = replace( + _reference("service", "permit-tenant"), + version="8", + ) + + derived = derive_service_restriction( + parent, + reference=reference, + temporal=temporal, + audience=("resident",), + channels=("portal",), + add_prerequisites=("tenant-residency",), + add_required_evidence_types=("identity",), + add_bindings=(ServiceBinding("workflow", "permit-review"),), + ) + + self.assertEqual(parent.reference, derived.derived_from_ref) + self.assertEqual(("resident",), derived.audience) + self.assertEqual(("portal",), derived.channels) + self.assertEqual( + ("registered-address", "tenant-residency"), + derived.prerequisites, + ) + self.assertIn(ServiceBinding("case", "permit-case"), derived.bindings) + self.assertEqual( + parent.availability_requirements, + derived.availability_requirements, + ) + self.assertEqual( + parent.availability_requirements, + ServiceDefinition.from_mapping( + derived.to_dict() + ).availability_requirements, + ) + with self.assertRaisesRegex(InstitutionalContextError, "widen.*audience"): + derive_service_restriction( + parent, + reference=reference, + temporal=temporal, + audience=("resident", "visitor"), + ) + with self.assertRaisesRegex(InstitutionalContextError, "beyond"): + derive_service_restriction( + parent, + reference=reference, + temporal=replace(temporal, valid_to=None), + ) + with self.assertRaisesRegex( + InstitutionalContextError, + "publication state", + ): + derive_service_restriction( + replace(parent, publication_state="suspended"), + reference=reference, + temporal=temporal, + publication_state="published", + ) + + def test_mandate_resolution_is_effective_scoped_and_deterministic(self) -> None: + context = _context() + mandate = MandateDefinition( + reference=_reference("mandate", "mandate-1"), + temporal=TemporalRevision( + revision="1", + valid_from=NOW - timedelta(days=1), + valid_to=NOW + timedelta(days=1), + recorded_at=NOW - timedelta(days=2), + ), + task_types=("committee.formal_decision",), + authority_types=("permit",), + organization_unit_refs=( + _reference("organization_unit", "unit-1"), + ), + function_refs=(_reference("function", "function-1"),), + jurisdiction_refs=context.jurisdiction_refs, + subject_types=("permit",), + evidence=context.evidence, + ) + request = MandateResolutionRequest( + tenant_id="tenant-1", + effective_at=NOW, + task_type="committee.formal_decision", + authority_type="permit", + organization_unit_ref=replace( + _reference("organization_unit", "unit-1"), version="9" + ), + function_ref=replace( + _reference("function", "function-1"), version="9" + ), + jurisdiction_refs=tuple( + replace(item, version="9") for item in context.jurisdiction_refs + ), + subject_type="permit", + ) + + resolution = resolve_mandate_candidates( + request, + ( + replace( + mandate, + reference=_reference("mandate", "retired"), + status="retired", + ), + mandate, + ), + ) + + self.assertTrue(resolution.competent) + self.assertEqual("mandate-1", resolution.mandates[0].reference.object_id) + self.assertEqual("file-1", resolution.evidence[0].evidence_id) + + ambiguous = resolve_mandate_candidates( + request, + ( + mandate, + replace( + mandate, + reference=_reference("mandate", "mandate-2"), + ), + ), + ) + self.assertFalse(ambiguous.competent) + self.assertEqual(2, len(ambiguous.conflict_refs)) + + missing_jurisdiction = resolve_mandate_candidates( + replace(request, jurisdiction_refs=()), + (mandate,), + ) + self.assertFalse(missing_jurisdiction.competent) + + def test_mandate_resolution_rejects_cross_tenant_candidates(self) -> None: + request = MandateResolutionRequest( + tenant_id="tenant-1", + effective_at=NOW, + task_type="committee.formal_decision", + authority_type="permit", + ) + candidate = MandateDefinition( + reference=_reference("mandate", "mandate-2", tenant_id="tenant-2"), + temporal=TemporalRevision(revision="1", recorded_at=NOW), + task_types=("committee.formal_decision",), + authority_types=("permit",), + ) + + with self.assertRaisesRegex(InstitutionalContextError, "cross tenants"): + resolve_mandate_candidates(request, (candidate,)) + + def test_mandate_lifecycle_revisions_are_linked_by_stable_identity(self) -> None: + current = MandateDefinition( + reference=_reference("mandate", "mandate-1"), + temporal=TemporalRevision(revision="1", recorded_at=NOW), + task_types=("committee.formal_decision",), + authority_types=("permit",), + status="active", + ) + temporal = TemporalRevision( + revision="2", + recorded_at=NOW + timedelta(minutes=1), + change_reason="Authority temporarily suspended.", + ) + + suspended = revise_mandate_definition( + current, + expected_revision="1", + temporal=temporal, + status="suspended", + suspension_reason="Delegation is under review.", + ) + + self.assertEqual("active", current.status) + self.assertEqual("suspended", suspended.status) + self.assertEqual("2", suspended.reference.version) + self.assertEqual(current.reference.object_id, suspended.reference.object_id) + with self.assertRaisesRegex(InstitutionalContextError, "stale"): + revise_mandate_definition( + current, + expected_revision="0", + temporal=temporal, + status="suspended", + suspension_reason="Review.", + ) + with self.assertRaisesRegex(InstitutionalContextError, "not allowed"): + revise_mandate_definition( + suspended, + expected_revision="2", + temporal=replace( + temporal, + revision="3", + change_reason="Invalid reset.", + ), + status="draft", + ) + + def test_formal_decision_revisions_are_linked_and_occ_guarded(self) -> None: + context = _context() + reference = replace( + _reference("decision", "decision-1"), + version=context.temporal.revision, + ) + context = replace(context, decision_ref=reference) + current = FormalDecision( + reference=reference, + temporal=context.temporal, + decision_type="permit", + subject_refs=(_reference("case", "case-1"),), + state="decided", + authority_context=context, + fact_evidence=context.evidence, + legal_bases=context.legal_bases, + operative_result="Permit granted.", + reasoning="Protected reasoning.", + requested_effects=( + DecisionEffectReference("notify", "requested"), + ), + ) + with self.assertRaisesRegex( + InstitutionalContextError, + "service account", + ): + replace(current, assurance_level="automated_under_mandate") + revision = TemporalRevision( + revision="8", + valid_from=NOW + timedelta(minutes=1), + recorded_at=NOW + timedelta(minutes=1), + change_reason="Decision entered into force.", + ) + + effective = revise_formal_decision( + current, + expected_revision="7", + temporal=revision, + state="effective", + ) + + self.assertEqual("decided", current.state) + self.assertEqual("8", effective.reference.version) + self.assertEqual("7", effective.supersedes_ref.version) + self.assertEqual( + effective.reference, + effective.authority_context.decision_ref, + ) + self.assertEqual(current.requested_effects, effective.requested_effects) + + with self.assertRaisesRegex(InstitutionalContextError, "stale"): + revise_formal_decision( + current, + expected_revision="6", + temporal=revision, + state="effective", + ) + with self.assertRaisesRegex(InstitutionalContextError, "not allowed"): + revise_formal_decision( + effective, + expected_revision="8", + temporal=replace( + revision, + revision="9", + change_reason="Invalid regression.", + ), + state="proposed", + ) + + corrected = revise_formal_decision( + effective, + expected_revision="8", + temporal=replace( + revision, + revision="9", + recorded_at=NOW + timedelta(minutes=2), + change_reason="Corrected operative wording.", + ), + state="corrected", + operative_result="Permit granted subject to condition A.", + ) + self.assertEqual(effective.reference, corrected.correction_of_ref) + self.assertEqual(effective.reference, corrected.supersedes_ref) + + def test_party_corrections_and_representation_revocation_are_versioned(self) -> None: + context = _context() + represented = _reference("party", "party-applicant") + representative = _reference("party", "party-agent") + representation = PartyRepresentation( + representative_party_ref=representative, + represented_party_ref=represented, + power_ref="power-1", + permitted_actions=("submit", "receive"), + temporal=TemporalRevision( + revision="1", + valid_from=NOW - timedelta(days=1), + recorded_at=NOW - timedelta(days=1), + ), + evidence=context.evidence, + ) + current = ProcedureParty( + reference=represented, + procedure_ref=_reference("case", "case-1"), + role="applicant", + subject=PartySubjectReference( + kind="identity", + provider="identity", + subject_id="identity-1", + tenant_id="tenant-1", + version="2", + ), + temporal=TemporalRevision(revision="1", recorded_at=NOW), + permitted_channels=("mail", "postbox"), + preferred_channels=("mail",), + delivery_recipient=True, + representations=(representation,), + contact_snapshot_refs=("addresses:snapshot-1",), + evidence=context.evidence, + ) + revision = TemporalRevision( + revision="2", + valid_from=NOW, + recorded_at=NOW + timedelta(minutes=1), + change_reason="Correct preferred delivery channel.", + ) + + corrected = revise_procedure_party( + current, + expected_revision="1", + temporal=revision, + preferred_channels=("postbox",), + ) + revoked_power = revoke_party_representation( + representation, + expected_revision="1", + temporal=TemporalRevision( + revision="2", + valid_from=NOW - timedelta(days=1), + valid_to=NOW + timedelta(minutes=2), + recorded_at=NOW + timedelta(minutes=2), + change_reason="Power was revoked.", + ), + revoked_at=NOW + timedelta(minutes=2), + ) + + self.assertEqual(("mail",), current.preferred_channels) + self.assertEqual(("postbox",), corrected.preferred_channels) + self.assertEqual("2", corrected.reference.version) + self.assertIsNone(representation.revoked_at) + self.assertEqual(NOW + timedelta(minutes=2), revoked_power.revoked_at) + with self.assertRaisesRegex(InstitutionalContextError, "stale"): + revise_procedure_party( + current, + expected_revision="0", + temporal=revision, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migration_lock.py b/tests/test_migration_lock.py new file mode 100644 index 0000000..cdbcd69 --- /dev/null +++ b/tests/test_migration_lock.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from govoplan_core.db.migration_lock import ( + deployment_migration_lock, + migration_lock_key, +) + + +class MigrationLockTests(unittest.TestCase): + def test_lock_key_is_stable_and_namespaced(self) -> None: + first = migration_lock_key("installation-a", "release") + self.assertEqual(first, migration_lock_key("installation-a", "release")) + self.assertNotEqual(first, migration_lock_key("installation-b", "release")) + self.assertNotEqual(first, migration_lock_key("installation-a", "dev")) + self.assertGreaterEqual(first, -(2**63)) + self.assertLess(first, 2**63) + + def test_non_postgres_migration_lock_is_a_noop(self) -> None: + entered = False + with deployment_migration_lock( + "sqlite+pysqlite:///:memory:", + installation_id="installation-a", + migration_track="release", + ): + entered = True + self.assertTrue(entered) + + def test_postgres_lock_is_held_for_the_guarded_operation(self) -> None: + result = MagicMock() + result.scalar_one.return_value = True + connection = MagicMock() + connection.execute.return_value = result + engine = MagicMock() + engine.connect.return_value = connection + with patch( + "govoplan_core.db.migration_lock.create_engine", + return_value=engine, + ): + with deployment_migration_lock( + "postgresql://user:secret@db.example.test/govoplan", + installation_id="installation-a", + migration_track="release", + ): + self.assertEqual(1, connection.execute.call_count) + + self.assertEqual(2, connection.execute.call_count) + connection.close.assert_called_once_with() + engine.dispose.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_module_system.py b/tests/test_module_system.py index 5f95c0e..e150853 100644 --- a/tests/test_module_system.py +++ b/tests/test_module_system.py @@ -1261,6 +1261,41 @@ finally: self.assertIn("module_active", {issue.code for issue in preflight.issues}) self.assertIn("module_desired", {issue.code for issue in preflight.issues}) + def test_module_installer_blocks_node_local_package_mutation_in_shared_profile( + self, + ) -> None: + plan = ModuleInstallPlan( + items=( + ModuleInstallPlanItem( + module_id="files", + action="update", + source="manual", + python_package="govoplan-files", + python_ref="govoplan-files==1.2.3", + ), + ) + ) + with patch.dict(os.environ, {"GOVOPLAN_STATE_PROFILE": "shared"}): + preflight = module_install_preflight( + plan=plan, + available={ + "files": ModuleManifest( + id="files", + name="Files", + version="1.2.2", + ) + }, + current_enabled=("files",), + desired_enabled=("files",), + maintenance_mode=True, + ) + + self.assertFalse(preflight.allowed) + self.assertIn( + "immutable_cluster_release_required", + {issue.code for issue in preflight.issues}, + ) + def test_module_installer_preflight_blocks_incompatible_manifest(self) -> None: root = Path(tempfile.mkdtemp(prefix="govoplan-installer-compat-", dir=_TEST_ROOT)) settings = _settings(root) diff --git a/tests/test_object_storage.py b/tests/test_object_storage.py new file mode 100644 index 0000000..a2a402d --- /dev/null +++ b/tests/test_object_storage.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +from io import BytesIO +from pathlib import Path +import tempfile +import unittest + +from govoplan_core.core.object_storage import ( + LocalFilesystemStorageBackend, + S3StorageBackend, + StorageBackendError, + StorageObjectMissing, + normalize_storage_key, +) + + +class _S3Error(RuntimeError): + def __init__(self, code: str, status: int) -> None: + super().__init__(code) + self.response = { + "Error": {"Code": code}, + "ResponseMetadata": {"HTTPStatusCode": status}, + } + + +class _FakeS3Client: + def __init__(self) -> None: + self.objects: dict[str, bytes] = {} + self.head_error: Exception | None = None + + def put_object(self, *, Key: str, Body: bytes, **_kwargs) -> None: + self.objects[Key] = Body + + def get_object(self, *, Key: str, **_kwargs): + try: + payload = self.objects[Key] + except KeyError as exc: + raise _S3Error("NoSuchKey", 404) from exc + return {"Body": BytesIO(payload), "ContentLength": len(payload)} + + def head_object(self, *, Key: str, **_kwargs): + if self.head_error is not None: + raise self.head_error + try: + payload = self.objects[Key] + except KeyError as exc: + raise _S3Error("NotFound", 404) from exc + return {"ContentLength": len(payload)} + + def delete_object(self, *, Key: str, **_kwargs) -> None: + self.objects.pop(Key, None) + + +class ObjectStorageTests(unittest.TestCase): + def test_local_backend_round_trip_pagination_and_safe_keys(self) -> None: + with tempfile.TemporaryDirectory(prefix="govoplan-object-store-") as directory: + backend = LocalFilesystemStorageBackend(Path(directory)) + backend.put_bytes("campaign/a.eml", b"a") + backend.put_bytes("campaign/b.eml", b"bb") + + first = backend.list_objects(prefix="campaign/", limit=1) + second = backend.list_objects( + prefix="campaign/", + after=first.next_cursor, + limit=1, + ) + + self.assertEqual(b"a", backend.get_bytes("campaign/a.eml")) + self.assertEqual(1, backend.stat("campaign/a.eml").size_bytes) + self.assertEqual( + ("campaign/a.eml",), tuple(item.key for item in first.objects) + ) + self.assertEqual("campaign/a.eml", first.next_cursor) + self.assertEqual( + ("campaign/b.eml",), tuple(item.key for item in second.objects) + ) + self.assertIsNone(second.next_cursor) + with self.assertRaises(StorageBackendError): + normalize_storage_key("../outside") + with self.assertRaises(StorageObjectMissing): + backend.get_bytes("campaign/missing.eml") + + def test_local_listing_is_globally_sorted_and_ignores_symlinks(self) -> None: + with ( + tempfile.TemporaryDirectory(prefix="govoplan-object-store-") as directory, + tempfile.TemporaryDirectory(prefix="govoplan-object-outside-") as outside, + ): + root = Path(directory) + backend = LocalFilesystemStorageBackend(root) + backend.put_bytes("a/x.txt", b"nested") + backend.put_bytes("a.txt", b"sibling") + backend.put_bytes("b.txt", b"last") + outside_file = Path(outside) / "secret.txt" + outside_file.write_bytes(b"outside") + (root / "linked-file.txt").symlink_to(outside_file) + (root / "linked-directory").symlink_to( + Path(outside), target_is_directory=True + ) + + first = backend.list_objects(prefix="", limit=2) + second = backend.list_objects( + prefix="", + after=first.next_cursor, + limit=2, + ) + + self.assertEqual( + ("a.txt", "a/x.txt"), + tuple(item.key for item in first.objects), + ) + self.assertEqual("a/x.txt", first.next_cursor) + self.assertEqual( + ("b.txt",), + tuple(item.key for item in second.objects), + ) + self.assertIsNone(second.next_cursor) + with self.assertRaises(StorageBackendError): + backend.exists("../outside") + + def test_s3_backend_distinguishes_missing_objects_from_backend_failure( + self, + ) -> None: + backend = S3StorageBackend( + bucket="files", + endpoint_url="http://garage:3900", + region_name="garage", + access_key_id="key", + secret_access_key="secret", + deployment_managed=True, + ) + client = _FakeS3Client() + backend._client = client + + backend.put_bytes("campaign/message.eml", b"message/rfc822") + self.assertTrue(backend.exists("campaign/message.eml")) + self.assertEqual(b"message/rfc822", backend.get_bytes("campaign/message.eml")) + self.assertFalse(backend.exists("campaign/missing.eml")) + + client.head_error = _S3Error("AccessDenied", 403) + with self.assertRaises(StorageBackendError): + backend.exists("campaign/message.eml") + + def test_external_s3_requires_explicit_https_deployment_trust(self) -> None: + trusted = S3StorageBackend( + bucket="files", + endpoint_url="https://s3.internal.example.test", + region_name="eu-test-1", + access_key_id="key", + secret_access_key="secret", + endpoint_trusted=True, + ) + trusted._client = _FakeS3Client() + trusted.put_bytes("files/document.txt", b"document") + self.assertEqual(b"document", trusted.get_bytes("files/document.txt")) + + rejected = S3StorageBackend( + bucket="files", + endpoint_url="http://s3.internal.example.test", + region_name="eu-test-1", + access_key_id="key", + secret_access_key="secret", + endpoint_trusted=True, + ) + with self.assertRaisesRegex(StorageBackendError, "HTTPS origin"): + rejected.client + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_provider_governance_contract.py b/tests/test_provider_governance_contract.py new file mode 100644 index 0000000..2a8aba2 --- /dev/null +++ b/tests/test_provider_governance_contract.py @@ -0,0 +1,536 @@ +from __future__ import annotations + +from datetime import UTC, datetime +import unittest + +from govoplan_core.core.modules import ModuleManifest +from govoplan_core.core.provider_governance import ( + ArchitectureDeclarationError, + ExternalProviderDeclaration, + ExternalProviderRuntimeState, + ExternalProviderStateContext, + ExternalProviderStateProviderRegistration, + ModuleArchitectureDeclaration, + ModuleArchitectureDocumentation, + ModuleMaturityEvidence, + ProviderBehaviorDeclaration, + ProviderObjectDeclaration, + collect_external_provider_states, + declared_module_architecture, + external_provider_from_mapping, + module_architecture_from_mapping, + module_architecture_issues, +) +from govoplan_core.core.registry import PlatformRegistry, RegistryError + + +def _architecture( + *, + providers: tuple[str, ...] = (), + modes: tuple[str, ...] = ("external_mirror",), +) -> ModuleArchitectureDeclaration: + return ModuleArchitectureDeclaration( + layer="data_reporting_integration", + kind="integration", + maturity="vertical_slice", + evidence=( + ModuleMaturityEvidence( + kind="test", + reference="tests/test_provider.py", + summary="Exercises the bounded provider contract.", + ), + ModuleMaturityEvidence( + kind="documentation", + reference="docs/PROVIDER.md", + summary="Documents authority and outage behavior.", + ), + ), + known_limits=("Only bounded test objects are supported.",), + supported_authority_modes=modes, + owned_concepts=("external transport",), + non_owned_concepts=("domain records",), + target_tested_providers=providers, + ) + + +def _mirror_provider() -> ExternalProviderDeclaration: + return ExternalProviderDeclaration( + id="connectors.read_mirror", + module_id="connectors", + label="Read-only mirror", + maturity="read", + operations=("discover", "search", "read", "preview"), + objects=( + ProviderObjectDeclaration( + object_type="register_entry", + field_groups=("identity", "content", "source_metadata"), + authority_modes=("external_authoritative", "external_mirror"), + default_authority_mode="external_mirror", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="ETag and source revision are retained.", + concurrency="Reads can require an expected source revision.", + freshness="The latest successful acquisition time is reported.", + health="Transport and parsing health are reported separately.", + max_read_items=500, + evidence="Immutable snapshots retain acquisition provenance.", + correction="Refresh creates a new snapshot; old evidence is retained.", + reconciliation="Compare the latest source revision and content digest.", + outage="The last governed snapshot remains readable as stale.", + classifications=("internal",), + purposes=("governed import",), + retention="Owning datasource retention policy applies.", + secret_handling="Credentials stay in credential envelopes.", + ), + capability_names=("connectors.readMirror",), + ) + + +def _writable_provider() -> ExternalProviderDeclaration: + return ExternalProviderDeclaration( + id="calendar.caldav_sync", + module_id="calendar", + label="CalDAV synchronization", + maturity="synchronize", + operations=("discover", "read", "write", "delete", "synchronize", "preview"), + objects=( + ProviderObjectDeclaration( + object_type="calendar_event", + field_groups=("identity", "schedule", "participants", "recurrence"), + authority_modes=("external_authoritative", "governed_sync"), + default_authority_mode="governed_sync", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="ETag and sync token.", + concurrency="Conditional requests reject stale ETags.", + freshness="Last successful sync and pending changes are reported.", + health="Discovery, authentication, and collection health are separate.", + max_read_items=1000, + idempotency="Stable UID and operation keys suppress duplicate effects.", + retry="Bounded retry is limited to classified transient failures.", + timeout_seconds=30, + conflicts="Conflicts remain explicit until policy or a user resolves them.", + outcome_unknown="Timed-out writes are outcome-unknown, never blindly retried.", + outcome_unknown_supported=True, + evidence="Request, response token, and reconciliation evidence are retained.", + audit_event_types=("calendar.sync.requested", "calendar.sync.reconciled"), + correction="A later governed write or tombstone corrects remote state.", + rollback="Remote writes are not assumed rollback-safe.", + compensation="A compensating update can be requested after reconciliation.", + reconciliation="Read by UID and compare ETag before retrying.", + outage="Local state remains marked stale and pending effects stay queued.", + classifications=("confidential",), + purposes=("calendar synchronization",), + retention="Calendar and audit retention policies apply independently.", + secret_handling="Credentials are referenced through credential envelopes.", + ), + capability_names=("calendar.sync",), + ) + + +def _state_registration( + *, + module_id: str, + provider_id: str, +) -> ExternalProviderStateProviderRegistration: + return ExternalProviderStateProviderRegistration( + module_id=module_id, + provider_id=provider_id, + provider=lambda _context: (), + ) + + +class ProviderGovernanceContractTests(unittest.TestCase): + def test_concise_declaration_keeps_repository_evidence_explicit(self) -> None: + architecture = declared_module_architecture( + layer="human_work_procedure", + kind="domain", + maturity="vertical_slice", + documentation_ref="docs/DOMAIN.md", + test_ref="tests/test_service.py", + known_limits=("Only the bounded service slice is implemented.",), + owned_concepts=("domain record",), + non_owned_concepts=("identity",), + ) + + self.assertEqual("vertical_slice", architecture.maturity) + self.assertEqual( + {"documentation", "test"}, + {item.kind for item in architecture.evidence}, + ) + self.assertEqual(("domain record",), architecture.owned_concepts) + + def test_declarations_round_trip_through_catalog_mappings(self) -> None: + provider = _writable_provider() + architecture = _architecture( + providers=(provider.id,), + modes=("external_authoritative", "governed_sync"), + ) + + self.assertEqual( + architecture, + module_architecture_from_mapping(architecture.to_dict()), + ) + self.assertEqual( + provider, + external_provider_from_mapping(provider.to_dict()), + ) + + def test_read_only_mirror_declaration_is_registry_validated(self) -> None: + provider = _mirror_provider() + registry = PlatformRegistry() + registry.register( + ModuleManifest( + id="connectors", + name="Connectors", + version="1.0.0", + architecture=_architecture( + providers=(provider.id,), + modes=("external_authoritative", "external_mirror"), + ), + external_providers=(provider,), + external_provider_state_providers=( + _state_registration( + module_id="connectors", + provider_id=provider.id, + ), + ), + capability_factories={"connectors.readMirror": lambda _context: object()}, + ) + ) + + snapshot = registry.validate() + + self.assertEqual("connectors", snapshot.manifests[0].id) + self.assertEqual((provider,), registry.external_provider_declarations()) + + def test_writable_outcome_unknown_provider_is_explicit(self) -> None: + provider = _writable_provider() + registry = PlatformRegistry() + registry.register( + ModuleManifest( + id="calendar", + name="Calendar", + version="1.0.0", + architecture=_architecture( + providers=(provider.id,), + modes=("external_authoritative", "governed_sync"), + ), + external_providers=(provider,), + external_provider_state_providers=( + _state_registration( + module_id="calendar", + provider_id=provider.id, + ), + ), + capability_factories={"calendar.sync": lambda _context: object()}, + ) + ) + + registry.validate() + + payload = provider.to_dict() + self.assertTrue(payload["behavior"]["outcome_unknown_supported"]) + self.assertIn("reconciliation", payload["behavior"]) + + def test_provider_requires_a_sanitized_runtime_state_registration(self) -> None: + provider = _mirror_provider() + registry = PlatformRegistry() + registry.register( + ModuleManifest( + id="connectors", + name="Connectors", + version="1.0.0", + architecture=_architecture( + providers=(provider.id,), + modes=("external_authoritative", "external_mirror"), + ), + external_providers=(provider,), + capability_factories={ + "connectors.readMirror": lambda _context: object() + }, + ) + ) + + with self.assertRaisesRegex( + RegistryError, + "require sanitized runtime state providers", + ): + registry.validate() + + def test_runtime_binding_states_are_validated_and_aggregated(self) -> None: + observed_at = datetime(2026, 8, 1, 12, 0, tzinfo=UTC) + + def states(_context: ExternalProviderStateContext): + return ( + ExternalProviderRuntimeState( + provider_id="calendar.caldav_sync", + binding_ref="calendar:sync-source:one", + authority_mode="governed_sync", + observed_at=observed_at, + configured=True, + active=True, + health="healthy", + freshness="current", + conflict="clear", + recovery="ready", + ), + ExternalProviderRuntimeState( + provider_id="calendar.caldav_sync", + binding_ref="calendar:sync-source:two", + authority_mode="external_mirror", + observed_at=observed_at, + configured=True, + active=True, + health="warning", + freshness="stale", + conflict="pending", + recovery="attention", + ), + ) + + payload = collect_external_provider_states( + ( + ExternalProviderStateProviderRegistration( + module_id="calendar", + provider_id="calendar.caldav_sync", + provider=states, + ), + ), + ExternalProviderStateContext(session=None, tenant_id="tenant-1"), + )["calendar.caldav_sync"] + + self.assertEqual("warning", payload["health"]) + self.assertEqual("stale", payload["freshness"]) + self.assertEqual("pending", payload["conflict"]) + self.assertEqual("attention", payload["recovery"]) + self.assertEqual(2, len(payload["bindings"])) + + def test_runtime_state_registration_requires_declared_provider(self) -> None: + registry = PlatformRegistry() + registry.register( + ModuleManifest( + id="calendar", + name="Calendar", + version="1.0.0", + architecture=_architecture(), + external_provider_state_providers=( + ExternalProviderStateProviderRegistration( + module_id="calendar", + provider_id="calendar.caldav_sync", + provider=lambda _context: (), + ), + ), + ) + ) + + with self.assertRaisesRegex(RegistryError, "undeclared provider"): + registry.validate() + + def test_runtime_state_failure_is_sanitized(self) -> None: + def fail(_context: ExternalProviderStateContext): + raise RuntimeError("secret transport detail") + + payload = collect_external_provider_states( + ( + ExternalProviderStateProviderRegistration( + module_id="calendar", + provider_id="calendar.caldav_sync", + provider=fail, + ), + ), + ExternalProviderStateContext(session=None), + )["calendar.caldav_sync"] + + self.assertEqual("error", payload["health"]) + self.assertNotIn("secret transport detail", str(payload)) + + def test_supported_claim_requires_evidence_not_only_a_string(self) -> None: + registry = PlatformRegistry() + registry.register( + ModuleManifest( + id="example", + name="Example", + version="1.0.0", + architecture=ModuleArchitectureDeclaration( + layer="domain_capability", + kind="domain", + maturity="supported", + evidence=(), + owned_concepts=("example",), + ), + ) + ) + + with self.assertRaisesRegex(RegistryError, "missing evidence"): + registry.validate() + + def test_supported_and_lts_claims_require_reference_packages(self) -> None: + for maturity in ("supported", "lts"): + with self.subTest(maturity=maturity): + declaration = ModuleArchitectureDeclaration( + layer="domain_capability", + kind="domain", + maturity=maturity, + evidence=tuple( + ModuleMaturityEvidence( + kind=kind, + reference=f"evidence/{kind}.json", + summary=f"{kind} evidence.", + ) + for kind in ( + "test", + "documentation", + "reference_process", + "target_test", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + "upgrade", + *(("compatibility",) if maturity == "lts" else ()), + ) + ), + owned_concepts=("example",), + documentation=ModuleArchitectureDocumentation( + upgrade=("docs/UPGRADE.md",), + recovery=("docs/RECOVERY.md",), + security=("docs/SECURITY.md",), + operations=("docs/OPERATIONS.md",), + ), + ) + + self.assertEqual( + (f"{maturity} maturity requires a reference package",), + module_architecture_issues(declaration, has_migrations=False), + ) + + def test_reference_ready_claim_requires_target_human_and_operator_evidence(self) -> None: + declaration = ModuleArchitectureDeclaration( + layer="domain_capability", + kind="domain", + maturity="reference_ready", + evidence=( + ModuleMaturityEvidence( + kind="test", + reference="tests/test_reference.py", + summary="Automated journey proof.", + ), + ModuleMaturityEvidence( + kind="documentation", + reference="docs/REFERENCE.md", + summary="Reference composition documentation.", + ), + ModuleMaturityEvidence( + kind="reference_process", + reference="docs/JOURNEY.md", + summary="Reference process definition.", + ), + ModuleMaturityEvidence( + kind="recovery", + reference="evidence/recovery.json", + summary="Recovery drill evidence.", + ), + ), + known_limits=("One provider profile remains bounded.",), + owned_concepts=("example",), + reference_packages=("reference.example",), + ) + + issues = module_architecture_issues(declaration, has_migrations=False) + + self.assertIn("accessibility", issues[0]) + self.assertIn("operations", issues[0]) + self.assertIn("privacy", issues[0]) + self.assertIn("security", issues[0]) + self.assertIn("target_test", issues[0]) + self.assertIn("missing documentation", issues[1]) + + def test_target_tested_provider_requires_provider_evidence(self) -> None: + evidence = tuple( + ModuleMaturityEvidence( + kind=kind, + reference=f"evidence/{kind}.json", + summary=f"{kind} evidence.", + ) + for kind in ( + "test", + "documentation", + "reference_process", + "target_test", + "recovery", + "security", + "operations", + "accessibility", + "privacy", + ) + ) + declaration = ModuleArchitectureDeclaration( + layer="data_reporting_integration", + kind="integration", + maturity="reference_ready", + evidence=evidence, + known_limits=("One bounded target profile is assessed.",), + reference_packages=("reference.integration",), + target_tested_providers=("connectors.example",), + documentation=ModuleArchitectureDocumentation( + recovery=("docs/RECOVERY.md",), + security=("docs/SECURITY.md",), + operations=("docs/OPERATIONS.md",), + ), + ) + + self.assertEqual( + ( + "maturity 'reference_ready' is missing evidence: provider", + ), + module_architecture_issues(declaration, has_migrations=False), + ) + + def test_provider_references_must_exist_in_manifest(self) -> None: + provider = _mirror_provider() + registry = PlatformRegistry() + registry.register( + ModuleManifest( + id="connectors", + name="Connectors", + version="1.0.0", + architecture=_architecture( + providers=(provider.id,), + modes=("external_authoritative", "external_mirror"), + ), + external_providers=(provider,), + ) + ) + + with self.assertRaisesRegex(RegistryError, "undeclared capabilities"): + registry.validate() + + def test_governed_sync_requires_read_and_write_semantics(self) -> None: + with self.assertRaisesRegex( + ArchitectureDeclarationError, + "read and write/synchronize", + ): + ExternalProviderDeclaration( + id="broken.sync", + module_id="broken", + label="Broken sync", + maturity="synchronize", + operations=("synchronize",), + objects=( + ProviderObjectDeclaration( + object_type="object", + field_groups=("content",), + authority_modes=("governed_sync",), + default_authority_mode="governed_sync", + ), + ), + behavior=_writable_provider().behavior, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_recovery_guarantees.py b/tests/test_recovery_guarantees.py new file mode 100644 index 0000000..5352ba5 --- /dev/null +++ b/tests/test_recovery_guarantees.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +import pytest + +from govoplan_core.core.recovery import ( + RecoveryCheckpoint, + RecoveryGuaranteeError, + RecoveryMode, + RecoveryOperation, + RecoveryPlan, + RecoveryStatus, + plan_recovery_operation, + prepare_recovery_operation, + record_recovery_checkpoint, + start_recovery_operation, + transition_recovery_operation, + verify_recovery_evidence_chain, +) +from govoplan_core.core.runtime_coordination import DistributedLease, acquire_lease +from govoplan_core.db.base import Base + + +def _session() -> tuple[Session, object]: + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[ + DistributedLease.__table__, + RecoveryOperation.__table__, + RecoveryCheckpoint.__table__, + ], + ) + return Session(engine, expire_on_commit=False), engine + + +def test_recovery_plan_requires_mode_specific_evidence() -> None: + with pytest.raises(RecoveryGuaranteeError, match="compensation steps"): + RecoveryPlan( + mode=RecoveryMode.COMPENSATION, + verification_steps=("verify",), + ).validate() + with pytest.raises(RecoveryGuaranteeError, match="backup reference"): + RecoveryPlan( + mode=RecoveryMode.SNAPSHOT_RESTORE, + verification_steps=("verify",), + ).validate() + with pytest.raises(RecoveryGuaranteeError, match="approval reference"): + RecoveryPlan( + mode=RecoveryMode.IRREVERSIBLE, + verification_steps=("verify",), + ).validate() + + +def test_recovery_plan_hashes_backup_and_approval_references() -> None: + plan = RecoveryPlan( + mode=RecoveryMode.SNAPSHOT_RESTORE, + verification_steps=("restore drill passed",), + backup_reference="backup://postgres/2026-08-01", + approval_reference="approval://change/42", + ) + + assert plan.as_dict()["backup_reference"] == "backup://postgres/2026-08-01" + assert plan.as_dict()["approval_reference"] == "approval://change/42" + + +def test_recovery_operation_records_verifiable_checkpoint_chain() -> None: + session, engine = _session() + try: + operation = plan_recovery_operation( + session, + installation_id="installation-1", + module_id="files", + operation_type="move-tree", + idempotency_key="move-42", + request={"source": "a", "target": "b"}, + recovery_plan=RecoveryPlan( + mode=RecoveryMode.COMPENSATION, + compensation_steps=("restore source links",), + verification_steps=("compare object inventory",), + ), + ) + prepare_recovery_operation( + session, + operation, + evidence={"inventory_sha256": "a" * 64}, + ) + start_recovery_operation(session, operation) + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RECOVERY_REQUIRED, + kind="side-effect-failed", + summary="Target index update failed", + evidence={"completed_steps": 1}, + failure_summary="target index unavailable", + ) + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RECOVERING, + kind="compensation-started", + summary="Compensation started", + ) + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RECOVERED, + kind="compensation-verified", + summary="Source inventory restored", + evidence={ + "verified": True, + "checks": {"object_inventory": "matched"}, + }, + ) + session.commit() + + assert operation.status == RecoveryStatus.RECOVERED.value + assert operation.checkpoint_count == 6 + assert verify_recovery_evidence_chain(session, operation.id) is True + finally: + session.close() + engine.dispose() + + +def test_recovery_transition_cannot_skip_preparation() -> None: + session, engine = _session() + try: + operation = plan_recovery_operation( + session, + installation_id="installation-1", + module_id="core", + operation_type="configuration-change", + idempotency_key="config-1", + request={"revision": 2}, + recovery_plan=RecoveryPlan( + mode=RecoveryMode.ATOMIC, + verification_steps=("read current revision",), + ), + ) + with pytest.raises(RecoveryGuaranteeError, match="not allowed"): + start_recovery_operation(session, operation) + finally: + session.close() + engine.dispose() + + +def test_non_atomic_failure_requires_explicit_recovery_and_verified_completion() -> ( + None +): + session, engine = _session() + try: + operation = plan_recovery_operation( + session, + installation_id="installation-1", + module_id="campaign", + operation_type="artifact-publish", + idempotency_key="publish-1", + request={"version": "version-1"}, + recovery_plan=RecoveryPlan( + mode=RecoveryMode.COMPENSATION, + compensation_steps=("delete published objects",), + verification_steps=("verify object inventory",), + ), + ) + prepare_recovery_operation( + session, + operation, + evidence={"preconditions": "verified"}, + ) + start_recovery_operation(session, operation) + + with pytest.raises(RecoveryGuaranteeError, match="cannot be marked failed"): + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.FAILED, + kind="failed", + summary="Object publication failed", + ) + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RECOVERY_REQUIRED, + kind="recovery-required", + summary="Published objects require compensation", + ) + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RECOVERING, + kind="compensation-started", + summary="Removing published objects", + ) + with pytest.raises(RecoveryGuaranteeError, match="verified evidence"): + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.RECOVERED, + kind="compensation-finished", + summary="Objects removed", + ) + finally: + session.close() + engine.dispose() + + +def test_recovery_evidence_rejects_plaintext_secrets() -> None: + session, engine = _session() + try: + with pytest.raises(RecoveryGuaranteeError, match="plaintext secrets"): + plan_recovery_operation( + session, + installation_id="installation-1", + module_id="mail", + operation_type="transport-change", + idempotency_key="transport-1", + request={"transport_id": "smtp-1"}, + metadata={"password": "do-not-store"}, + recovery_plan=RecoveryPlan( + mode=RecoveryMode.ATOMIC, + verification_steps=("read persisted transport",), + ), + ) + finally: + session.close() + engine.dispose() + + +def test_fenced_recovery_rejects_low_level_mutation_without_claim() -> None: + session, engine = _session() + try: + claim = acquire_lease( + session, + installation_id="installation-1", + resource_key="files:move-tree", + holder_node_id="worker-1", + holder_incarnation="worker-1-incarnation", + ttl_seconds=300, + now=datetime.now(timezone.utc), + ) + assert claim is not None + operation = plan_recovery_operation( + session, + installation_id="installation-1", + module_id="files", + operation_type="move-tree", + idempotency_key="move-fenced-1", + request={"source": "a", "target": "b"}, + recovery_plan=RecoveryPlan( + mode=RecoveryMode.COMPENSATION, + compensation_steps=("restore source links",), + verification_steps=("compare object inventory",), + ), + lease_claim=claim, + ) + + with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"): + transition_recovery_operation( + session, + operation, + status=RecoveryStatus.PREPARED, + kind="prepared", + summary="Attempted without fence", + evidence={"inventory_sha256": "a" * 64}, + ) + with pytest.raises(RecoveryGuaranteeError, match="distributed lease fence"): + record_recovery_checkpoint( + session, + operation, + kind="manual-note", + summary="Attempted without fence", + evidence={}, + ) + + prepared = prepare_recovery_operation( + session, + operation, + evidence={"inventory_sha256": "a" * 64}, + lease_claim=claim, + ) + assert prepared.status == RecoveryStatus.PREPARED.value + finally: + session.close() + engine.dispose() diff --git a/tests/test_runtime_agents.py b/tests/test_runtime_agents.py new file mode 100644 index 0000000..9e34045 --- /dev/null +++ b/tests/test_runtime_agents.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from govoplan_core.core.runtime_coordination import ( + RuntimeCoordinationError, + RuntimeIdentity, +) +from govoplan_core.server.runtime_agent import RuntimeNodeAgent + + +class _Session: + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def commit(self) -> None: + return None + + +class _Database: + def SessionLocal(self) -> _Session: + return _Session() + + +class _Consumer: + def __init__(self) -> None: + self.cancelled: list[str] = [] + self.added: list[str] = [] + + def cancel_task_queue(self, queue: str) -> None: + self.cancelled.append(queue) + + def add_task_queue(self, queue: str) -> None: + self.added.append(queue) + + +def test_api_runtime_agent_fails_readiness_on_heartbeat_error() -> None: + agent = RuntimeNodeAgent( + settings=SimpleNamespace( + installation_id="installation-1", + runtime_role="api", + runtime_node_id="api-1", + runtime_heartbeat_seconds=15, + celery_queues="", + ), + software_version="0.1.14", + module_ids=("core",), + ) + agent.coordination_healthy = True + + def fail() -> None: + raise RuntimeCoordinationError("database unavailable") + + agent._heartbeat = fail + assert asyncio.run(agent._heartbeat_once()) is False + assert agent.coordination_healthy is False + + agent._heartbeat = lambda: None + assert asyncio.run(agent._heartbeat_once()) is True + assert agent.coordination_healthy is True + + +def test_worker_disables_consumers_without_reclaiming_stale_identity( + monkeypatch, +) -> None: + from govoplan_core import celery_app + + identity = RuntimeIdentity( + installation_id="installation-1", + node_id="worker-1", + incarnation="stale-incarnation", + role="worker", + software_version="0.1.14", + composition_hash="a" * 64, + queues=("default", "mail"), + ) + consumer = _Consumer() + original_state = ( + celery_app._worker_identity, + celery_app._worker_consumer, + celery_app._worker_draining, + ) + celery_app._worker_identity = identity + celery_app._worker_consumer = consumer + celery_app._worker_draining = False + monkeypatch.setattr(celery_app, "get_database", lambda: _Database()) + + def reject_heartbeat(*_args, **_kwargs): + raise RuntimeCoordinationError("stale node incarnation") + + monkeypatch.setattr( + celery_app, + "heartbeat_runtime_node", + reject_heartbeat, + ) + monkeypatch.setattr( + celery_app, + "register_runtime_node", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("heartbeat failures must not reclaim an identity") + ), + ) + try: + celery_app._heartbeat_worker_runtime() + assert consumer.cancelled == ["default", "mail"] + assert celery_app._worker_draining is True + + monkeypatch.setattr( + celery_app, + "heartbeat_runtime_node", + lambda *_args, **_kwargs: SimpleNamespace(state="active"), + ) + celery_app._heartbeat_worker_runtime() + assert consumer.added == ["default", "mail"] + assert celery_app._worker_draining is False + finally: + ( + celery_app._worker_identity, + celery_app._worker_consumer, + celery_app._worker_draining, + ) = original_state diff --git a/tests/test_runtime_coordination.py b/tests/test_runtime_coordination.py new file mode 100644 index 0000000..7143425 --- /dev/null +++ b/tests/test_runtime_coordination.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from govoplan_core.core.runtime_coordination import ( + DistributedLease, + RuntimeIdentity, + RuntimeNode, + RuntimeNodeState, + StaleFence, + acquire_lease, + assert_lease_fence, + heartbeat_runtime_node, + list_runtime_nodes, + register_runtime_node, + request_runtime_node_drain, +) +from govoplan_core.db.base import Base + + +def _session() -> tuple[Session, object]: + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[RuntimeNode.__table__, DistributedLease.__table__], + ) + return Session(engine, expire_on_commit=False), engine + + +def _identity(*, node_id: str, incarnation: str) -> RuntimeIdentity: + return RuntimeIdentity( + installation_id="installation-1", + node_id=node_id, + incarnation=incarnation, + role="worker", + software_version="0.1.14", + composition_hash="a" * 64, + queues=("default",), + ) + + +def test_node_incarnation_guards_heartbeat_and_drain_state() -> None: + session, engine = _session() + now = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) + first = _identity(node_id="worker-1", incarnation="first") + replacement = _identity(node_id="worker-1", incarnation="replacement") + try: + register_runtime_node(session, first, now=now) + heartbeat_runtime_node(session, first, now=now + timedelta(seconds=5)) + register_runtime_node(session, replacement, now=now + timedelta(seconds=10)) + + with pytest.raises(RuntimeError, match="stale node incarnation"): + heartbeat_runtime_node(session, first, now=now + timedelta(seconds=15)) + + node = request_runtime_node_drain( + session, + installation_id="installation-1", + node_id="worker-1", + reason="rolling update", + now=now + timedelta(seconds=20), + ) + assert node.state == RuntimeNodeState.DRAINING.value + rows = list_runtime_nodes( + session, + installation_id="installation-1", + stale_after_seconds=60, + now=now + timedelta(seconds=25), + ) + assert rows[0]["incarnation"] == "replacement" + assert rows[0]["state"] == "draining" + assert rows[0]["stale"] is False + finally: + session.close() + engine.dispose() + + +def test_expired_lease_reassignment_increments_fence() -> None: + session, engine = _session() + now = datetime(2026, 8, 1, 10, 0, tzinfo=timezone.utc) + try: + first = acquire_lease( + session, + installation_id="installation-1", + resource_key="migration:release", + holder_node_id="node-a", + holder_incarnation="a1", + ttl_seconds=30, + now=now, + ) + assert first is not None and first.fencing_token == 1 + blocked = acquire_lease( + session, + installation_id="installation-1", + resource_key="migration:release", + holder_node_id="node-b", + holder_incarnation="b1", + ttl_seconds=30, + now=now + timedelta(seconds=10), + ) + assert blocked is None + + replacement = acquire_lease( + session, + installation_id="installation-1", + resource_key="migration:release", + holder_node_id="node-b", + holder_incarnation="b1", + ttl_seconds=30, + now=now + timedelta(seconds=31), + ) + assert replacement is not None + assert replacement.fencing_token == 2 + with pytest.raises(StaleFence): + assert_lease_fence( + session, + first, + now=now + timedelta(seconds=32), + ) + assert_lease_fence( + session, + replacement, + now=now + timedelta(seconds=32), + ) + finally: + session.close() + engine.dispose() diff --git a/tests/test_wait_for_database.py b/tests/test_wait_for_database.py new file mode 100644 index 0000000..695d04f --- /dev/null +++ b/tests/test_wait_for_database.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from govoplan_core.commands.wait_for_database import main + + +class WaitForDatabaseTests(unittest.TestCase): + def test_waits_until_exact_configured_heads_are_visible(self) -> None: + with ( + patch( + "govoplan_core.commands.wait_for_database.configured_migration_heads", + return_value=("head-a", "head-b"), + ), + patch( + "govoplan_core.commands.wait_for_database.database_migration_heads", + side_effect=[("head-a",), ("head-a", "head-b")], + ), + patch("govoplan_core.commands.wait_for_database.time.sleep"), + ): + result = main(["--timeout-seconds", "1", "--poll-seconds", "0.01"]) + + self.assertEqual(0, result) + + def test_returns_temporary_failure_when_heads_do_not_match(self) -> None: + with ( + patch( + "govoplan_core.commands.wait_for_database.configured_migration_heads", + return_value=("head-b",), + ), + patch( + "govoplan_core.commands.wait_for_database.database_migration_heads", + return_value=("head-a",), + ), + ): + result = main(["--timeout-seconds", "0"]) + + self.assertEqual(75, result) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/package-lock.json b/webui/package-lock.json index a1dc35d..c7f3e1c 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -14,19 +14,26 @@ "@govoplan/audit-webui": "file:../../govoplan-audit/webui", "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", + "@govoplan/cases-webui": "file:../../govoplan-cases/webui", + "@govoplan/committee-webui": "file:../../govoplan-committee/webui", "@govoplan/dashboard-webui": "file:../../govoplan-dashboard/webui", "@govoplan/dataflow-webui": "file:../../govoplan-dataflow/webui", "@govoplan/datasources-webui": "file:../../govoplan-datasources/webui", "@govoplan/dist-lists-webui": "file:../../govoplan-dist-lists/webui", "@govoplan/docs-webui": "file:../../govoplan-docs/webui", "@govoplan/files-webui": "file:../../govoplan-files/webui", + "@govoplan/forms-runtime-webui": "file:../../govoplan-forms-runtime/webui", + "@govoplan/forms-webui": "file:../../govoplan-forms/webui", "@govoplan/idm-webui": "file:../../govoplan-idm/webui", "@govoplan/mail-webui": "file:../../govoplan-mail/webui", "@govoplan/notifications-webui": "file:../../govoplan-notifications/webui", "@govoplan/ops-webui": "file:../../govoplan-ops/webui", "@govoplan/organizations-webui": "file:../../govoplan-organizations/webui", "@govoplan/policy-webui": "file:../../govoplan-policy/webui", + "@govoplan/portal-webui": "file:../../govoplan-portal/webui", "@govoplan/postbox-webui": "file:../../govoplan-postbox/webui", + "@govoplan/projects-webui": "file:../../govoplan-projects/webui", + "@govoplan/reporting-webui": "file:../../govoplan-reporting/webui", "@govoplan/risk-compliance-webui": "file:../../govoplan-risk-compliance/webui", "@govoplan/scheduling-webui": "file:../../govoplan-scheduling/webui", "@govoplan/search-webui": "file:../../govoplan-search/webui", @@ -170,6 +177,38 @@ } } }, + "../../govoplan-cases/webui": { + "name": "@govoplan/cases-webui", + "version": "0.1.8", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "../../govoplan-committee/webui": { + "name": "@govoplan/committee-webui", + "version": "0.1.8", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-dashboard/webui": { "name": "@govoplan/dashboard-webui", "version": "0.1.8", @@ -276,6 +315,37 @@ } } }, + "../../govoplan-forms-runtime/webui": { + "name": "@govoplan/forms-runtime-webui", + "version": "0.1.14", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "../../govoplan-forms/webui": { + "name": "@govoplan/forms-webui", + "version": "0.1.14", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-idm/webui": { "name": "@govoplan/idm-webui", "version": "0.1.8", @@ -387,6 +457,22 @@ } } }, + "../../govoplan-portal/webui": { + "name": "@govoplan/portal-webui", + "version": "0.1.8", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-postbox/webui": { "name": "@govoplan/postbox-webui", "version": "0.1.2", @@ -403,6 +489,38 @@ } } }, + "../../govoplan-projects/webui": { + "name": "@govoplan/projects-webui", + "version": "0.1.14", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, + "../../govoplan-reporting/webui": { + "name": "@govoplan/reporting-webui", + "version": "0.1.14", + "peerDependencies": { + "@govoplan/core-webui": "^0.1.14", + "lucide-react": "^1.23.0", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9" + }, + "peerDependenciesMeta": { + "@govoplan/core-webui": { + "optional": true + } + } + }, "../../govoplan-risk-compliance/webui": { "name": "@govoplan/risk-compliance-webui", "version": "0.1.8", @@ -1279,6 +1397,14 @@ "resolved": "../../govoplan-campaign/webui", "link": true }, + "node_modules/@govoplan/cases-webui": { + "resolved": "../../govoplan-cases/webui", + "link": true + }, + "node_modules/@govoplan/committee-webui": { + "resolved": "../../govoplan-committee/webui", + "link": true + }, "node_modules/@govoplan/dashboard-webui": { "resolved": "../../govoplan-dashboard/webui", "link": true @@ -1303,6 +1429,14 @@ "resolved": "../../govoplan-files/webui", "link": true }, + "node_modules/@govoplan/forms-runtime-webui": { + "resolved": "../../govoplan-forms-runtime/webui", + "link": true + }, + "node_modules/@govoplan/forms-webui": { + "resolved": "../../govoplan-forms/webui", + "link": true + }, "node_modules/@govoplan/idm-webui": { "resolved": "../../govoplan-idm/webui", "link": true @@ -1327,10 +1461,22 @@ "resolved": "../../govoplan-policy/webui", "link": true }, + "node_modules/@govoplan/portal-webui": { + "resolved": "../../govoplan-portal/webui", + "link": true + }, "node_modules/@govoplan/postbox-webui": { "resolved": "../../govoplan-postbox/webui", "link": true }, + "node_modules/@govoplan/projects-webui": { + "resolved": "../../govoplan-projects/webui", + "link": true + }, + "node_modules/@govoplan/reporting-webui": { + "resolved": "../../govoplan-reporting/webui", + "link": true + }, "node_modules/@govoplan/risk-compliance-webui": { "resolved": "../../govoplan-risk-compliance/webui", "link": true diff --git a/webui/package.json b/webui/package.json index 7d8970d..0e1db4e 100644 --- a/webui/package.json +++ b/webui/package.json @@ -53,20 +53,27 @@ "@govoplan/admin-webui": "file:../../govoplan-admin/webui", "@govoplan/audit-webui": "file:../../govoplan-audit/webui", "@govoplan/calendar-webui": "file:../../govoplan-calendar/webui", + "@govoplan/cases-webui": "file:../../govoplan-cases/webui", "@govoplan/campaign-webui": "file:../../govoplan-campaign/webui", + "@govoplan/committee-webui": "file:../../govoplan-committee/webui", "@govoplan/dashboard-webui": "file:../../govoplan-dashboard/webui", "@govoplan/dataflow-webui": "file:../../govoplan-dataflow/webui", "@govoplan/datasources-webui": "file:../../govoplan-datasources/webui", "@govoplan/dist-lists-webui": "file:../../govoplan-dist-lists/webui", "@govoplan/docs-webui": "file:../../govoplan-docs/webui", "@govoplan/files-webui": "file:../../govoplan-files/webui", + "@govoplan/forms-webui": "file:../../govoplan-forms/webui", + "@govoplan/forms-runtime-webui": "file:../../govoplan-forms-runtime/webui", "@govoplan/idm-webui": "file:../../govoplan-idm/webui", "@govoplan/mail-webui": "file:../../govoplan-mail/webui", "@govoplan/notifications-webui": "file:../../govoplan-notifications/webui", "@govoplan/ops-webui": "file:../../govoplan-ops/webui", "@govoplan/organizations-webui": "file:../../govoplan-organizations/webui", "@govoplan/policy-webui": "file:../../govoplan-policy/webui", + "@govoplan/portal-webui": "file:../../govoplan-portal/webui", "@govoplan/postbox-webui": "file:../../govoplan-postbox/webui", + "@govoplan/projects-webui": "file:../../govoplan-projects/webui", + "@govoplan/reporting-webui": "file:../../govoplan-reporting/webui", "@govoplan/risk-compliance-webui": "file:../../govoplan-risk-compliance/webui", "@govoplan/scheduling-webui": "file:../../govoplan-scheduling/webui", "@govoplan/search-webui": "file:../../govoplan-search/webui", diff --git a/webui/scripts/test-module-permutations.mjs b/webui/scripts/test-module-permutations.mjs index d5700c0..9994322 100644 --- a/webui/scripts/test-module-permutations.mjs +++ b/webui/scripts/test-module-permutations.mjs @@ -7,20 +7,27 @@ const packageByModule = { addresses: "@govoplan/addresses-webui", audit: "@govoplan/audit-webui", calendar: "@govoplan/calendar-webui", + cases: "@govoplan/cases-webui", campaigns: "@govoplan/campaign-webui", + committee: "@govoplan/committee-webui", dashboard: "@govoplan/dashboard-webui", dataflow: "@govoplan/dataflow-webui", datasources: "@govoplan/datasources-webui", dist_lists: "@govoplan/dist-lists-webui", docs: "@govoplan/docs-webui", files: "@govoplan/files-webui", + forms: "@govoplan/forms-webui", + forms_runtime: "@govoplan/forms-runtime-webui", idm: "@govoplan/idm-webui", mail: "@govoplan/mail-webui", notifications: "@govoplan/notifications-webui", organizations: "@govoplan/organizations-webui", ops: "@govoplan/ops-webui", policy: "@govoplan/policy-webui", + portal: "@govoplan/portal-webui", postbox: "@govoplan/postbox-webui", + projects: "@govoplan/projects-webui", + reporting: "@govoplan/reporting-webui", risk_compliance: "@govoplan/risk-compliance-webui", scheduling: "@govoplan/scheduling-webui", search: "@govoplan/search-webui", @@ -49,12 +56,19 @@ const cases = [ { name: "views-only", modules: ["views"] }, { name: "views-with-administration", modules: ["access", "admin", "views"] }, { name: "calendar-only", modules: ["calendar"] }, + { name: "cases-only", modules: ["cases"] }, + { name: "committee-only", modules: ["committee"] }, { name: "files-only", modules: ["files"] }, + { name: "forms-only", modules: ["forms"] }, + { name: "forms-runtime", modules: ["forms", "forms_runtime"] }, { name: "mail-only", modules: ["mail"] }, { name: "notifications-only", modules: ["notifications"] }, { name: "organizations-only", modules: ["organizations"] }, { name: "idm-with-organizations", modules: ["organizations", "idm"] }, { name: "postbox-only", modules: ["postbox"] }, + { name: "portal-only", modules: ["portal"] }, + { name: "projects-only", modules: ["projects"] }, + { name: "reporting-only", modules: ["reporting"] }, { name: "campaign-only", modules: ["campaigns"] }, { name: "campaign-with-postbox", modules: ["campaigns", "postbox"] }, { name: "campaign-with-files-no-mail", modules: ["campaigns", "files"] }, @@ -64,7 +78,7 @@ const cases = [ { name: "search-only", modules: ["search"] }, { name: "risk-compliance-only", modules: ["risk_compliance"] }, { name: "docs-and-ops", modules: ["access", "docs", "ops"] }, - { name: "full-product", modules: ["access", "tenancy", "admin", "addresses", "policy", "audit", "dashboard", "datasources", "dataflow", "dist_lists", "workflow", "views", "organizations", "idm", "campaigns", "files", "mail", "notifications", "docs", "ops", "calendar", "scheduling", "postbox", "risk_compliance", "search"] } + { name: "full-product", modules: ["access", "tenancy", "admin", "addresses", "policy", "audit", "dashboard", "datasources", "dataflow", "dist_lists", "workflow", "views", "organizations", "idm", "cases", "committee", "campaigns", "files", "forms", "forms_runtime", "mail", "notifications", "docs", "ops", "calendar", "scheduling", "portal", "postbox", "projects", "reporting", "risk_compliance", "search"] } ]; const npmExec = process.env.npm_execpath; diff --git a/webui/src/platform/modules.ts b/webui/src/platform/modules.ts index edf961e..fed47ca 100644 --- a/webui/src/platform/modules.ts +++ b/webui/src/platform/modules.ts @@ -1,4 +1,4 @@ -import { Activity, Bell, BookUser, Building2, CalendarClock, CalendarDays, ClipboardPenLine, DatabaseZap, Folder, Form, Inbox, LayoutDashboard, LayoutTemplate, ListTree, Mail, Mails, RadioTower, Shield, ShieldCheck, Users, Waypoints, Workflow as WorkflowIcon, type LucideIcon } from "lucide-react"; +import { Activity, Bell, BookUser, BriefcaseBusiness, Building2, CalendarClock, CalendarDays, ClipboardPenLine, DatabaseZap, Folder, FolderKanban, Form, Gavel, Inbox, Landmark, LayoutDashboard, LayoutTemplate, ListTree, Mail, Mails, RadioTower, Shield, ShieldCheck, Users, Waypoints, Workflow as WorkflowIcon, type LucideIcon } from "lucide-react"; import installedWebModuleLoaderSource from "virtual:govoplan-installed-modules"; import type { AuthInfo, DashboardWidgetContribution, DashboardWidgetsUiCapability, EffectiveViewProjection, PlatformModuleInfo, PlatformNavItem, PlatformPublicModuleInfo, PlatformViewSurface, PlatformWebModule } from "../types"; import { @@ -56,6 +56,7 @@ const iconByName: Record = { admin: Shield, bell: Bell, "book-user": BookUser, + "briefcase-business": BriefcaseBusiness, building: Building2, "building-2": Building2, calendar: CalendarDays, @@ -67,8 +68,11 @@ const iconByName: Record = { file: Folder, files: Folder, folder: Folder, + "folder-kanban": FolderKanban, form: Form, + gavel: Gavel, inbox: Inbox, + landmark: Landmark, "layout-template": LayoutTemplate, "list-tree": ListTree, mail: Mail, diff --git a/webui/src/types.ts b/webui/src/types.ts index 7062e45..ca58048 100644 --- a/webui/src/types.ts +++ b/webui/src/types.ts @@ -1057,6 +1057,31 @@ export type PlatformModuleInfo = { dependencies: string[]; optional_dependencies: string[]; enabled: boolean; + architecture?: { + contract_version: string; + layer: string; + kind: string; + maturity: string; + evidence: Array<{ kind: string; reference: string; summary: string }>; + known_limits: string[]; + supported_authority_modes: string[]; + owned_concepts: string[]; + non_owned_concepts: string[]; + reference_packages: string[]; + target_tested_providers: string[]; + documentation: Record; + } | null; + external_providers?: Array<{ + contract_version: string; + id: string; + module_id: string; + label: string; + maturity: string; + operations: string[]; + authority_modes: string[]; + objects: Array>; + behavior: Record; + }>; runtime_ui_capabilities?: string[]; nav: Array<{ path: string; diff --git a/webui/vite.config.ts b/webui/vite.config.ts index 37d494e..564435f 100644 --- a/webui/vite.config.ts +++ b/webui/vite.config.ts @@ -18,19 +18,26 @@ const defaultWebModulePackages = [ "@govoplan/addresses-webui", "@govoplan/audit-webui", "@govoplan/calendar-webui", + "@govoplan/cases-webui", "@govoplan/campaign-webui", + "@govoplan/committee-webui", "@govoplan/dataflow-webui", "@govoplan/datasources-webui", "@govoplan/dashboard-webui", "@govoplan/docs-webui", "@govoplan/files-webui", + "@govoplan/forms-webui", + "@govoplan/forms-runtime-webui", "@govoplan/idm-webui", "@govoplan/mail-webui", "@govoplan/notifications-webui", "@govoplan/organizations-webui", "@govoplan/ops-webui", "@govoplan/policy-webui", + "@govoplan/portal-webui", "@govoplan/postbox-webui", + "@govoplan/projects-webui", + "@govoplan/reporting-webui", "@govoplan/risk-compliance-webui", "@govoplan/scheduling-webui", "@govoplan/search-webui", @@ -221,11 +228,14 @@ export default defineConfig({ fileURLToPath(new URL('../../govoplan-addresses/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-audit/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-calendar/webui', import.meta.url)), + fileURLToPath(new URL('../../govoplan-cases/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-dataflow/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-datasources/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-dashboard/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-docs/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-files/webui', import.meta.url)), + fileURLToPath(new URL('../../govoplan-forms/webui', import.meta.url)), + fileURLToPath(new URL('../../govoplan-forms-runtime/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-idm/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-mail/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-notifications/webui', import.meta.url)), @@ -233,6 +243,7 @@ export default defineConfig({ fileURLToPath(new URL('../../govoplan-campaign/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-ops/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-policy/webui', import.meta.url)), + fileURLToPath(new URL('../../govoplan-portal/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-postbox/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-risk-compliance/webui', import.meta.url)), fileURLToPath(new URL('../../govoplan-scheduling/webui', import.meta.url)),