from __future__ import annotations from pathlib import Path from govoplan_core.core.access import ( CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER, ) from govoplan_core.core.datasources import ( CAPABILITY_DATASOURCE_CATALOGUE, CAPABILITY_DATASOURCE_LIFECYCLE, CAPABILITY_DATASOURCE_ORIGINS, CAPABILITY_DATASOURCE_PUBLICATION, ) from govoplan_core.core.module_guards import ( drop_table_retirement_provider, persistent_table_uninstall_guard, ) from govoplan_core.core.modules import ( DocumentationTopic, FrontendModule, FrontendRoute, MigrationSpec, ModuleContext, ModuleInterfaceProvider, ModuleInterfaceRequirement, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate, ) from govoplan_core.core.provider_governance import ( ModuleArchitectureDeclaration, ModuleArchitectureDocumentation, ModuleMaturityEvidence, ) from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base from govoplan_datasources.backend.db import models as datasource_models from govoplan_datasources.backend.service import ( ADMIN_SCOPE, CATALOGUE_READ_SCOPE, SOURCE_WRITE_SCOPE, STAGE_WRITE_SCOPE, SqlDatasourceProvider, ) MODULE_ID = "datasources" MODULE_NAME = "Datasources" MODULE_VERSION = "0.1.14" DATASOURCE_INTERFACE_VERSION = "0.1.0" ARCHITECTURE = ModuleArchitectureDeclaration( layer="data_reporting_integration", kind="foundation", maturity="vertical_slice", evidence=( ModuleMaturityEvidence( kind="test", reference="tests/test_lifecycle.py", summary="Exercises staging, immutable materialization, publication, and typed governance behavior.", ), ModuleMaturityEvidence( kind="documentation", reference="docs/CONCEPT.md", summary="Defines datasource ownership and connector/dataflow boundaries.", ), ModuleMaturityEvidence( kind="migration", reference="tests/test_migrations.py", summary="Migrates and validates the governed catalogue schema.", ), ), known_limits=( "Governance references are stable provider-neutral refs; dedicated selectors depend on the owning optional modules.", "Quality and freshness policies are stored and snapshotted but enforcement remains provider-specific.", ), supported_authority_modes=( "native_authoritative", "external_authoritative", "external_mirror", "governed_sync", "governance_overlay", "linked_reference", ), owned_concepts=( "datasource catalogue identity", "datasource governance metadata", "staging and immutable materializations", "datasource publication lifecycle", ), non_owned_concepts=( "external transport and credentials", "dataflow transformation semantics", "report presentation", ), documentation=ModuleArchitectureDocumentation( migration=("tests/test_migrations.py",), recovery=("docs/CONCEPT.md",), security=("docs/CONCEPT.md",), operations=("docs/CONCEPT.md",), ), ) def _permission(scope: str, label: str, description: str) -> PermissionDefinition: module_id, resource, action = scope.split(":", 2) return PermissionDefinition( scope=scope, label=label, description=description, category="Datasources", level="tenant", module_id=module_id, resource=resource, action=action, ) PERMISSIONS = ( _permission( CATALOGUE_READ_SCOPE, "View datasources", "Discover and preview policy-visible datasources and materializations.", ), _permission( SOURCE_WRITE_SCOPE, "Manage datasources", "Register, refresh, freeze, and retire governed datasources.", ), _permission( STAGE_WRITE_SCOPE, "Stage datasource content", "Upload, validate, inspect, and promote bounded datasource stages.", ), _permission( ADMIN_SCOPE, "Administer datasources", "Manage every tenant datasource, stage, materialization, and lifecycle policy.", ), ) ROLE_TEMPLATES = ( RoleTemplate( slug="datasource_manager", name="Datasource manager", description="Register, stage, refresh, freeze, and retire governed data.", permissions=( CATALOGUE_READ_SCOPE, SOURCE_WRITE_SCOPE, STAGE_WRITE_SCOPE, ), ), RoleTemplate( slug="datasource_reader", name="Datasource reader", description="Discover and preview policy-visible datasource states.", permissions=(CATALOGUE_READ_SCOPE,), ), ) def _router(context: ModuleContext): from govoplan_datasources.backend.runtime import configure_runtime configure_runtime(registry=context.registry) from govoplan_datasources.backend.router import router return router def _provider(context: ModuleContext) -> SqlDatasourceProvider: return SqlDatasourceProvider(registry=context.registry) def _tenant_summary(session, tenant_id: str) -> dict[str, int]: return { "datasources": ( session.query(datasource_models.DatasourceRecord) .filter( datasource_models.DatasourceRecord.tenant_id == tenant_id, datasource_models.DatasourceRecord.deleted_at.is_(None), ) .count() ), "datasource_stages_ready": ( session.query(datasource_models.DatasourceStageRecord) .filter( datasource_models.DatasourceStageRecord.tenant_id == tenant_id, datasource_models.DatasourceStageRecord.state == "ready", ) .count() ), "datasource_frozen_states": ( session.query(datasource_models.DatasourceMaterializationRecord) .filter( datasource_models.DatasourceMaterializationRecord.tenant_id == tenant_id, datasource_models.DatasourceMaterializationRecord.frozen_at.is_not( None ), ) .count() ), } manifest = ModuleManifest( id=MODULE_ID, name=MODULE_NAME, version=MODULE_VERSION, dependencies=(), optional_dependencies=( "access", "audit", "connectors", "files", "notifications", "policy", ), optional_capabilities=( CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_DATASOURCE_ORIGINS, ), provides_interfaces=( ModuleInterfaceProvider( name="datasources.catalogue", version=DATASOURCE_INTERFACE_VERSION, ), ModuleInterfaceProvider( name="datasources.lifecycle", version=DATASOURCE_INTERFACE_VERSION, ), ModuleInterfaceProvider( name="datasources.materializations", version=DATASOURCE_INTERFACE_VERSION, ), ModuleInterfaceProvider( name="datasources.staging", version=DATASOURCE_INTERFACE_VERSION, ), ModuleInterfaceProvider( name="datasources.publication", version=DATASOURCE_INTERFACE_VERSION, ), ), requires_interfaces=( ModuleInterfaceRequirement( name="connectors.datasource_origins", version_min="0.1.0", version_max_exclusive="1.0.0", optional=True, ), ), permissions=PERMISSIONS, role_templates=ROLE_TEMPLATES, nav_items=( NavItem( path="/datasources", label="Datasources", icon="database-zap", required_any=(CATALOGUE_READ_SCOPE, ADMIN_SCOPE), order=70, ), ), frontend=FrontendModule( module_id=MODULE_ID, package_name="@govoplan/datasources-webui", routes=( FrontendRoute( path="/datasources", component="DatasourcesPage", required_any=(CATALOGUE_READ_SCOPE, ADMIN_SCOPE), order=70, ), ), nav_items=( NavItem( path="/datasources", label="Datasources", icon="database-zap", required_any=(CATALOGUE_READ_SCOPE, ADMIN_SCOPE), order=70, ), ), view_surfaces=( ViewSurface(id="datasources.page", module_id=MODULE_ID, kind="route", label="Datasources", order=70), ViewSurface(id="datasources.catalogue", module_id=MODULE_ID, kind="section", label="Datasource catalogue", order=10), ViewSurface(id="datasources.staging", module_id=MODULE_ID, kind="section", label="Datasource staging", order=20), ViewSurface(id="datasources.origins", module_id=MODULE_ID, kind="section", label="Datasource origins", order=30), ViewSurface(id="datasources.governance", module_id=MODULE_ID, kind="action", label="Datasource governance", order=40), ViewSurface(id="datasources.preview", module_id=MODULE_ID, kind="section", label="Datasource preview and materializations", order=50), ), ), route_factory=_router, capability_factories={ CAPABILITY_DATASOURCE_CATALOGUE: _provider, CAPABILITY_DATASOURCE_LIFECYCLE: _provider, CAPABILITY_DATASOURCE_PUBLICATION: _provider, }, tenant_summary_providers=(_tenant_summary,), migration_spec=MigrationSpec( module_id=MODULE_ID, metadata=Base.metadata, script_location=str(Path(__file__).with_name("migrations") / "versions"), retirement_supported=True, retirement_provider=drop_table_retirement_provider( datasource_models.DatasourcePublicationRecord, datasource_models.DatasourceStageRecord, datasource_models.DatasourceMaterializationRecord, datasource_models.DatasourcePayloadRowRecord, datasource_models.DatasourcePayloadRecord, datasource_models.DatasourceRecord, label="Datasources", ), retirement_notes=( "Destructive retirement drops datasource catalogue entries, immutable " "materializations, and staging evidence after a database snapshot." ), ), uninstall_guard_providers=( persistent_table_uninstall_guard( datasource_models.DatasourceRecord, datasource_models.DatasourceMaterializationRecord, datasource_models.DatasourcePayloadRecord, datasource_models.DatasourcePayloadRowRecord, datasource_models.DatasourceStageRecord, datasource_models.DatasourcePublicationRecord, label="Datasources", ), ), architecture=ARCHITECTURE, documentation=( DocumentationTopic( id="datasources.lifecycle", title="Datasource lifecycle", summary="Governed live, cached, and static data/register entries with staging, frozen states, provenance, and institutional ownership context.", body=( "Datasources is the provider-neutral catalogue consumed by Dataflow, " "Workflow, Reporting, and policy-aware modules. Static data is staged " "before promotion. Cached data refreshes connector origins into immutable " "materializations. Live data is read through a connector and may be frozen " "for reproducible evidence. Connectors owns protocols and credentials; " "Datasources owns data identity, provenance, lifecycle, read semantics, " "and the typed governance catalogue for authority, purpose, quality, " "freshness, classification, correction, and dependent services, flows, " "reports, controls, and decisions. Governance metadata visibility does not " "grant access to protected rows." ), layer="available", documentation_types=("admin", "user"), audience=("operator", "module_admin", "power_user", "product_owner"), related_modules=( "connectors", "dataflow", "workflow_engine", "files", "reporting", "risk_compliance", ), order=70, metadata={ "seed": True, "help_contexts": [ "datasources.page", "datasources.catalogue", "datasources.staging", "datasources.origins", "datasources.preview", ], }, ), DocumentationTopic( id="datasources.governance", title="Datasource authority and governance", summary="Explain who owns data meaning, authority, correction, privacy, quality, freshness, retention, and dependent uses.", body=( "Authority mode states whether GovOPlaN, an external system, a synchronized projection, an overlay, or a linked reference " "controls the data. The authoritative source, owner, steward, responsible organization/function, schema owner, privacy " "profile, retention policy, transfer agreement, legal basis, holds, correction procedure, purposes, official keys, and " "known limits provide discoverable institutional context. Freshness and quality policies are typed JSON contracts retained " "with materialization evidence; enforcement remains with the provider or consuming control that declares support. Metadata " "visibility never grants row access." ), layer="available", documentation_types=("admin", "user"), audience=("operator", "module_admin", "data_steward", "product_owner"), related_modules=("policy", "organizations", "idm", "dataflow", "reporting", "risk_compliance"), order=71, metadata={ "seed": True, "help_contexts": [ "datasources.governance", "datasources.field.authority-mode", "datasources.field.authoritative-source", "datasources.field.classification", "datasources.field.publication-state", "datasources.field.freshness-policy", "datasources.field.quality-policy", ], }, ), DocumentationTopic( id="datasources.reference.fields-and-consequences", title="Datasource fields and lifecycle consequences", summary="Live, cached, static, staging, promotion, refresh, freeze, and retirement semantics.", body=( "A Datasource key is the stable catalogue identity used by consumers. Live mode reads through an available origin; cached " "mode refreshes an origin into immutable revisions; static mode promotes uploaded content from staging. Stages are bounded, " "inspectable, and non-consumable until promoted. Promotion creates or updates a governed Datasource and appends an immutable " "materialization. Refresh appends a new cached revision without rewriting older evidence. Freeze labels an immutable, " "addressable state for reproducible execution. Retirement removes the Datasource from new definitions while retained " "materialization references remain governed. Connector absence disables origin registration but leaves local catalogue and " "staging behavior available." ), layer="available", documentation_types=("admin", "user"), audience=("operator", "module_admin", "power_user", "product_owner"), related_modules=("connectors", "dataflow", "workflow_engine", "reporting", "audit"), order=72, metadata={ "seed": True, "help_contexts": [ "datasources.field.origin", "datasources.field.key", "datasources.field.mode", "datasources.action.promote", "datasources.action.refresh", "datasources.action.freeze", "datasources.action.retire", ], "consequence_classes": { "promote": "Creates or updates a governed Datasource and appends an immutable materialization revision.", "refresh": "Reads the cached origin and appends a new immutable materialization revision.", "freeze": "Creates a labelled immutable state for reproducible consumers and evidence.", "retire": "Prevents new selection while retained materialization references remain governed.", }, }, ), ), ) def get_manifest() -> ModuleManifest: return manifest __all__ = [ "DATASOURCE_INTERFACE_VERSION", "MODULE_ID", "MODULE_VERSION", "get_manifest", "manifest", ]