from __future__ import annotations from pathlib import Path from govoplan_core.core.module_guards import ( drop_table_retirement_provider, persistent_table_uninstall_guard, ) from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_RUN_LIFECYCLE from govoplan_core.core.modules import ( DocumentationTopic, FrontendModule, MigrationSpec, ModuleInterfaceProvider, ModuleInterfaceRequirement, ModuleContext, ModuleManifest, NavItem, PermissionDefinition, RoleTemplate, ) from govoplan_core.core.datasources import ( CAPABILITY_DATASOURCE_CATALOGUE, CAPABILITY_DATASOURCE_LIFECYCLE, CAPABILITY_DATASOURCE_PUBLICATION, ) from govoplan_core.db.base import Base from govoplan_dataflow.backend.db import models as dataflow_models MODULE_ID = "dataflow" MODULE_NAME = "Dataflow" MODULE_VERSION = "0.1.14" READ_SCOPE = "dataflow:pipeline:read" WRITE_SCOPE = "dataflow:pipeline:write" RUN_SCOPE = "dataflow:pipeline:run" ADMIN_SCOPE = "dataflow:pipeline:admin" def _permission(scope: str, label: str, description: str) -> PermissionDefinition: module_id, resource, action = scope.split(":", 2) return PermissionDefinition( scope=scope, label=label, description=description, category="Dataflow", level="tenant", module_id=module_id, resource=resource, action=action, ) PERMISSIONS = ( _permission( READ_SCOPE, "View data pipelines", "Read pipeline definitions, revisions, validation diagnostics, and run summaries.", ), _permission( WRITE_SCOPE, "Manage data pipelines", "Create, edit, version, archive, and remove data pipeline definitions.", ), _permission( RUN_SCOPE, "Run data pipelines", "Validate and execute bounded previews or approved pipeline runs.", ), _permission( ADMIN_SCOPE, "Administer Dataflow", "Manage every tenant pipeline and future execution, retention, and publication policies.", ), ) ROLE_TEMPLATES = ( RoleTemplate( slug="dataflow_designer", name="Dataflow designer", description="Design, validate, and preview governed data pipelines.", permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE), ), RoleTemplate( slug="dataflow_operator", name="Dataflow operator", description="Inspect and run approved data pipelines without changing their definitions.", permissions=(READ_SCOPE, RUN_SCOPE), ), RoleTemplate( slug="dataflow_viewer", name="Dataflow viewer", description="Inspect pipeline definitions, revisions, diagnostics, and run summaries.", permissions=(READ_SCOPE,), ), ) DOCUMENTATION = ( DocumentationTopic( id="dataflow.module-boundary", title="Dataflow module boundary", summary="Versioned tabular transformations with graphical and constrained SQL editing.", body=( "Dataflow owns canonical pipeline graphs, immutable revisions, validation, constrained " "SQL compilation, preview and run diagnostics, and lineage references. Datasources owns " "the governed catalogue and materializations, while Connectors owns external acquisition " "and credentials; Reporting owns analytical presentation and exports; " "Workflow owns orchestration and human handoffs; Risk Compliance owns sanctions review " "semantics and policy gates. User SQL is compiled into approved transforms and is never " "passed unchecked to a backing database." ), layer="available", documentation_types=("admin", "user"), audience=("operator", "module_admin", "power_user", "product_owner"), order=75, related_modules=( "datasources", "connectors", "files", "reporting", "workflow", "risk_compliance", "notifications", "policy", "audit", ), metadata={ "first_slice": ( "Inline and governed datasources, union, join, filter, deduplication, select, " "derived columns, aggregate, sort, limit, output, revisioning, and bounded preview." ), "sql_safety": "Constrained AST compilation only; no pass-through execution.", }, ), ) def _dataflow_router(context: ModuleContext): from govoplan_dataflow.backend.runtime import configure_runtime configure_runtime(registry=context.registry, settings=context.settings) from govoplan_dataflow.backend.router import router return router def _run_provider(context: ModuleContext): from govoplan_dataflow.backend.service import SqlDataflowRunLifecycleProvider return SqlDataflowRunLifecycleProvider(registry=context.registry) def _tenant_summary(session, tenant_id: str) -> dict[str, int]: return { "dataflow_pipelines": ( session.query(dataflow_models.DataflowPipeline) .filter( dataflow_models.DataflowPipeline.tenant_id == tenant_id, dataflow_models.DataflowPipeline.deleted_at.is_(None), ) .count() ), "dataflow_runs": ( session.query(dataflow_models.DataflowRun) .filter(dataflow_models.DataflowRun.tenant_id == tenant_id) .count() ), } manifest = ModuleManifest( id=MODULE_ID, name=MODULE_NAME, version=MODULE_VERSION, dependencies=(), optional_dependencies=( "access", "audit", "datasources", "files", "notifications", "policy", "reporting", "risk_compliance", "workflow", ), optional_capabilities=( CAPABILITY_DATASOURCE_CATALOGUE, CAPABILITY_DATASOURCE_LIFECYCLE, CAPABILITY_DATASOURCE_PUBLICATION, ), provides_interfaces=( ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION), ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION), ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION), ModuleInterfaceProvider(name="dataflow.dataset_output", version=MODULE_VERSION), ), requires_interfaces=( ModuleInterfaceRequirement( name="datasources.catalogue", version_min="0.1.0", version_max_exclusive="1.0.0", optional=True, ), ModuleInterfaceRequirement( name="datasources.lifecycle", version_min="0.1.0", version_max_exclusive="1.0.0", optional=True, ), ModuleInterfaceRequirement( name="datasources.publication", version_min="0.1.0", version_max_exclusive="1.0.0", optional=True, ), ), permissions=PERMISSIONS, role_templates=ROLE_TEMPLATES, nav_items=( NavItem( path="/dataflow", label="Dataflow", icon="waypoints", required_any=(READ_SCOPE, ADMIN_SCOPE), order=72, ), ), frontend=FrontendModule( module_id=MODULE_ID, package_name="@govoplan/dataflow-webui", nav_items=( NavItem( path="/dataflow", label="Dataflow", icon="waypoints", required_any=(READ_SCOPE, ADMIN_SCOPE), order=72, ), ), ), route_factory=_dataflow_router, capability_factories={ CAPABILITY_DATAFLOW_RUN_LIFECYCLE: _run_provider, }, tenant_summary_providers=(_tenant_summary,), migration_spec=MigrationSpec( module_id=MODULE_ID, metadata=Base.metadata, script_location=str(Path(__file__).with_name("migrations") / "versions"), retirement_supported=True, retirement_provider=drop_table_retirement_provider( dataflow_models.DataflowRun, dataflow_models.DataflowPipelineRevision, dataflow_models.DataflowPipeline, label="Dataflow", ), retirement_notes=( "Destructive retirement drops Dataflow definitions, revisions, and run evidence " "after the installer captures a database snapshot." ), ), uninstall_guard_providers=( persistent_table_uninstall_guard( dataflow_models.DataflowPipeline, dataflow_models.DataflowPipelineRevision, dataflow_models.DataflowRun, label="Dataflow", ), ), documentation=DOCUMENTATION, ) def get_manifest() -> ModuleManifest: return manifest __all__ = [ "ADMIN_SCOPE", "MODULE_ID", "MODULE_NAME", "MODULE_VERSION", "READ_SCOPE", "RUN_SCOPE", "WRITE_SCOPE", "get_manifest", "manifest", ]