273 lines
8.2 KiB
Python
273 lines
8.2 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from govoplan_core.core.module_guards import (
|
|
drop_table_retirement_provider,
|
|
persistent_table_uninstall_guard,
|
|
)
|
|
from govoplan_core.core.modules import (
|
|
DocumentationTopic,
|
|
FrontendModule,
|
|
MigrationSpec,
|
|
ModuleInterfaceProvider,
|
|
ModuleInterfaceRequirement,
|
|
ModuleContext,
|
|
ModuleManifest,
|
|
NavItem,
|
|
PermissionDefinition,
|
|
RoleTemplate,
|
|
)
|
|
from govoplan_core.core.datasources import (
|
|
CAPABILITY_DATASOURCE_CATALOGUE,
|
|
CAPABILITY_DATASOURCE_LIFECYCLE,
|
|
)
|
|
from govoplan_core.db.base import Base
|
|
from govoplan_dataflow.backend.db import models as dataflow_models
|
|
|
|
|
|
MODULE_ID = "dataflow"
|
|
MODULE_NAME = "Dataflow"
|
|
MODULE_VERSION = "0.1.14"
|
|
|
|
READ_SCOPE = "dataflow:pipeline:read"
|
|
WRITE_SCOPE = "dataflow:pipeline:write"
|
|
RUN_SCOPE = "dataflow:pipeline:run"
|
|
ADMIN_SCOPE = "dataflow:pipeline:admin"
|
|
|
|
|
|
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
|
|
module_id, resource, action = scope.split(":", 2)
|
|
return PermissionDefinition(
|
|
scope=scope,
|
|
label=label,
|
|
description=description,
|
|
category="Dataflow",
|
|
level="tenant",
|
|
module_id=module_id,
|
|
resource=resource,
|
|
action=action,
|
|
)
|
|
|
|
|
|
PERMISSIONS = (
|
|
_permission(
|
|
READ_SCOPE,
|
|
"View data pipelines",
|
|
"Read pipeline definitions, revisions, validation diagnostics, and run summaries.",
|
|
),
|
|
_permission(
|
|
WRITE_SCOPE,
|
|
"Manage data pipelines",
|
|
"Create, edit, version, archive, and remove data pipeline definitions.",
|
|
),
|
|
_permission(
|
|
RUN_SCOPE,
|
|
"Run data pipelines",
|
|
"Validate and execute bounded previews or approved pipeline runs.",
|
|
),
|
|
_permission(
|
|
ADMIN_SCOPE,
|
|
"Administer Dataflow",
|
|
"Manage every tenant pipeline and future execution, retention, and publication policies.",
|
|
),
|
|
)
|
|
|
|
ROLE_TEMPLATES = (
|
|
RoleTemplate(
|
|
slug="dataflow_designer",
|
|
name="Dataflow designer",
|
|
description="Design, validate, and preview governed data pipelines.",
|
|
permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE),
|
|
),
|
|
RoleTemplate(
|
|
slug="dataflow_operator",
|
|
name="Dataflow operator",
|
|
description="Inspect and run approved data pipelines without changing their definitions.",
|
|
permissions=(READ_SCOPE, RUN_SCOPE),
|
|
),
|
|
RoleTemplate(
|
|
slug="dataflow_viewer",
|
|
name="Dataflow viewer",
|
|
description="Inspect pipeline definitions, revisions, diagnostics, and run summaries.",
|
|
permissions=(READ_SCOPE,),
|
|
),
|
|
)
|
|
|
|
DOCUMENTATION = (
|
|
DocumentationTopic(
|
|
id="dataflow.module-boundary",
|
|
title="Dataflow module boundary",
|
|
summary="Versioned tabular transformations with graphical and constrained SQL editing.",
|
|
body=(
|
|
"Dataflow owns canonical pipeline graphs, immutable revisions, validation, constrained "
|
|
"SQL compilation, preview and run diagnostics, and lineage references. Datasources owns "
|
|
"the governed catalogue and materializations, while Connectors owns external acquisition "
|
|
"and credentials; Reporting owns analytical presentation and exports; "
|
|
"Workflow owns orchestration and human handoffs; Risk Compliance owns sanctions review "
|
|
"semantics and policy gates. User SQL is compiled into approved transforms and is never "
|
|
"passed unchecked to a backing database."
|
|
),
|
|
layer="available",
|
|
documentation_types=("admin", "user"),
|
|
audience=("operator", "module_admin", "power_user", "product_owner"),
|
|
order=75,
|
|
related_modules=(
|
|
"datasources",
|
|
"connectors",
|
|
"files",
|
|
"reporting",
|
|
"workflow",
|
|
"risk_compliance",
|
|
"notifications",
|
|
"policy",
|
|
"audit",
|
|
),
|
|
metadata={
|
|
"first_slice": (
|
|
"Inline and governed datasources, union, join, filter, deduplication, select, "
|
|
"derived columns, aggregate, sort, limit, output, revisioning, and bounded preview."
|
|
),
|
|
"sql_safety": "Constrained AST compilation only; no pass-through execution.",
|
|
},
|
|
),
|
|
)
|
|
|
|
|
|
def _dataflow_router(context: ModuleContext):
|
|
from govoplan_dataflow.backend.runtime import configure_runtime
|
|
|
|
configure_runtime(registry=context.registry, settings=context.settings)
|
|
from govoplan_dataflow.backend.router import router
|
|
|
|
return router
|
|
|
|
|
|
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
|
return {
|
|
"dataflow_pipelines": (
|
|
session.query(dataflow_models.DataflowPipeline)
|
|
.filter(
|
|
dataflow_models.DataflowPipeline.tenant_id == tenant_id,
|
|
dataflow_models.DataflowPipeline.deleted_at.is_(None),
|
|
)
|
|
.count()
|
|
),
|
|
"dataflow_runs": (
|
|
session.query(dataflow_models.DataflowRun)
|
|
.filter(dataflow_models.DataflowRun.tenant_id == tenant_id)
|
|
.count()
|
|
),
|
|
}
|
|
|
|
|
|
manifest = ModuleManifest(
|
|
id=MODULE_ID,
|
|
name=MODULE_NAME,
|
|
version=MODULE_VERSION,
|
|
dependencies=(),
|
|
optional_dependencies=(
|
|
"access",
|
|
"audit",
|
|
"datasources",
|
|
"files",
|
|
"notifications",
|
|
"policy",
|
|
"reporting",
|
|
"risk_compliance",
|
|
"workflow",
|
|
),
|
|
optional_capabilities=(
|
|
CAPABILITY_DATASOURCE_CATALOGUE,
|
|
CAPABILITY_DATASOURCE_LIFECYCLE,
|
|
),
|
|
provides_interfaces=(
|
|
ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION),
|
|
ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION),
|
|
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
|
|
ModuleInterfaceProvider(name="dataflow.dataset_output", version=MODULE_VERSION),
|
|
),
|
|
requires_interfaces=(
|
|
ModuleInterfaceRequirement(
|
|
name="datasources.catalogue",
|
|
version_min="0.1.0",
|
|
version_max_exclusive="1.0.0",
|
|
optional=True,
|
|
),
|
|
ModuleInterfaceRequirement(
|
|
name="datasources.lifecycle",
|
|
version_min="0.1.0",
|
|
version_max_exclusive="1.0.0",
|
|
optional=True,
|
|
),
|
|
),
|
|
permissions=PERMISSIONS,
|
|
role_templates=ROLE_TEMPLATES,
|
|
nav_items=(
|
|
NavItem(
|
|
path="/dataflow",
|
|
label="Dataflow",
|
|
icon="waypoints",
|
|
required_any=(READ_SCOPE, ADMIN_SCOPE),
|
|
order=72,
|
|
),
|
|
),
|
|
frontend=FrontendModule(
|
|
module_id=MODULE_ID,
|
|
package_name="@govoplan/dataflow-webui",
|
|
nav_items=(
|
|
NavItem(
|
|
path="/dataflow",
|
|
label="Dataflow",
|
|
icon="waypoints",
|
|
required_any=(READ_SCOPE, ADMIN_SCOPE),
|
|
order=72,
|
|
),
|
|
),
|
|
),
|
|
route_factory=_dataflow_router,
|
|
tenant_summary_providers=(_tenant_summary,),
|
|
migration_spec=MigrationSpec(
|
|
module_id=MODULE_ID,
|
|
metadata=Base.metadata,
|
|
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
|
retirement_supported=True,
|
|
retirement_provider=drop_table_retirement_provider(
|
|
dataflow_models.DataflowRun,
|
|
dataflow_models.DataflowPipelineRevision,
|
|
dataflow_models.DataflowPipeline,
|
|
label="Dataflow",
|
|
),
|
|
retirement_notes=(
|
|
"Destructive retirement drops Dataflow definitions, revisions, and run evidence "
|
|
"after the installer captures a database snapshot."
|
|
),
|
|
),
|
|
uninstall_guard_providers=(
|
|
persistent_table_uninstall_guard(
|
|
dataflow_models.DataflowPipeline,
|
|
dataflow_models.DataflowPipelineRevision,
|
|
dataflow_models.DataflowRun,
|
|
label="Dataflow",
|
|
),
|
|
),
|
|
documentation=DOCUMENTATION,
|
|
)
|
|
|
|
|
|
def get_manifest() -> ModuleManifest:
|
|
return manifest
|
|
|
|
|
|
__all__ = [
|
|
"ADMIN_SCOPE",
|
|
"MODULE_ID",
|
|
"MODULE_NAME",
|
|
"MODULE_VERSION",
|
|
"READ_SCOPE",
|
|
"RUN_SCOPE",
|
|
"WRITE_SCOPE",
|
|
"get_manifest",
|
|
"manifest",
|
|
]
|