Initialize governed Dataflow module
This commit is contained in:
240
src/govoplan_dataflow/backend/manifest.py
Normal file
240
src/govoplan_dataflow/backend/manifest.py
Normal file
@@ -0,0 +1,240 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from govoplan_core.core.module_guards import (
|
||||
drop_table_retirement_provider,
|
||||
persistent_table_uninstall_guard,
|
||||
)
|
||||
from govoplan_core.core.modules import (
|
||||
DocumentationTopic,
|
||||
FrontendModule,
|
||||
MigrationSpec,
|
||||
ModuleInterfaceProvider,
|
||||
ModuleManifest,
|
||||
NavItem,
|
||||
PermissionDefinition,
|
||||
RoleTemplate,
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_dataflow.backend.db import models as dataflow_models
|
||||
|
||||
|
||||
MODULE_ID = "dataflow"
|
||||
MODULE_NAME = "Dataflow"
|
||||
MODULE_VERSION = "0.1.14"
|
||||
|
||||
READ_SCOPE = "dataflow:pipeline:read"
|
||||
WRITE_SCOPE = "dataflow:pipeline:write"
|
||||
RUN_SCOPE = "dataflow:pipeline:run"
|
||||
ADMIN_SCOPE = "dataflow:pipeline:admin"
|
||||
|
||||
|
||||
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
|
||||
module_id, resource, action = scope.split(":", 2)
|
||||
return PermissionDefinition(
|
||||
scope=scope,
|
||||
label=label,
|
||||
description=description,
|
||||
category="Dataflow",
|
||||
level="tenant",
|
||||
module_id=module_id,
|
||||
resource=resource,
|
||||
action=action,
|
||||
)
|
||||
|
||||
|
||||
PERMISSIONS = (
|
||||
_permission(
|
||||
READ_SCOPE,
|
||||
"View data pipelines",
|
||||
"Read pipeline definitions, revisions, validation diagnostics, and run summaries.",
|
||||
),
|
||||
_permission(
|
||||
WRITE_SCOPE,
|
||||
"Manage data pipelines",
|
||||
"Create, edit, version, archive, and remove data pipeline definitions.",
|
||||
),
|
||||
_permission(
|
||||
RUN_SCOPE,
|
||||
"Run data pipelines",
|
||||
"Validate and execute bounded previews or approved pipeline runs.",
|
||||
),
|
||||
_permission(
|
||||
ADMIN_SCOPE,
|
||||
"Administer Dataflow",
|
||||
"Manage every tenant pipeline and future execution, retention, and publication policies.",
|
||||
),
|
||||
)
|
||||
|
||||
ROLE_TEMPLATES = (
|
||||
RoleTemplate(
|
||||
slug="dataflow_designer",
|
||||
name="Dataflow designer",
|
||||
description="Design, validate, and preview governed data pipelines.",
|
||||
permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE),
|
||||
),
|
||||
RoleTemplate(
|
||||
slug="dataflow_operator",
|
||||
name="Dataflow operator",
|
||||
description="Inspect and run approved data pipelines without changing their definitions.",
|
||||
permissions=(READ_SCOPE, RUN_SCOPE),
|
||||
),
|
||||
RoleTemplate(
|
||||
slug="dataflow_viewer",
|
||||
name="Dataflow viewer",
|
||||
description="Inspect pipeline definitions, revisions, diagnostics, and run summaries.",
|
||||
permissions=(READ_SCOPE,),
|
||||
),
|
||||
)
|
||||
|
||||
DOCUMENTATION = (
|
||||
DocumentationTopic(
|
||||
id="dataflow.module-boundary",
|
||||
title="Dataflow module boundary",
|
||||
summary="Versioned tabular transformations with graphical and constrained SQL editing.",
|
||||
body=(
|
||||
"Dataflow owns canonical pipeline graphs, immutable revisions, validation, constrained "
|
||||
"SQL compilation, preview and run diagnostics, and lineage references. Connectors owns "
|
||||
"source connections and credentials; Reporting owns analytical presentation and exports; "
|
||||
"Workflow owns orchestration and human handoffs; Risk Compliance owns sanctions review "
|
||||
"semantics and policy gates. User SQL is compiled into approved transforms and is never "
|
||||
"passed unchecked to a backing database."
|
||||
),
|
||||
layer="available",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("operator", "module_admin", "power_user", "product_owner"),
|
||||
order=75,
|
||||
related_modules=(
|
||||
"connectors",
|
||||
"files",
|
||||
"reporting",
|
||||
"workflow",
|
||||
"risk_compliance",
|
||||
"notifications",
|
||||
"policy",
|
||||
"audit",
|
||||
),
|
||||
metadata={
|
||||
"first_slice": "Inline source, filter, select, aggregate, sort, limit, output, revisioning, and bounded preview.",
|
||||
"sql_safety": "Constrained AST compilation only; no pass-through execution.",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _dataflow_router(_context):
|
||||
from govoplan_dataflow.backend.router import router
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
||||
return {
|
||||
"dataflow_pipelines": (
|
||||
session.query(dataflow_models.DataflowPipeline)
|
||||
.filter(
|
||||
dataflow_models.DataflowPipeline.tenant_id == tenant_id,
|
||||
dataflow_models.DataflowPipeline.deleted_at.is_(None),
|
||||
)
|
||||
.count()
|
||||
),
|
||||
"dataflow_runs": (
|
||||
session.query(dataflow_models.DataflowRun)
|
||||
.filter(dataflow_models.DataflowRun.tenant_id == tenant_id)
|
||||
.count()
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
manifest = ModuleManifest(
|
||||
id=MODULE_ID,
|
||||
name=MODULE_NAME,
|
||||
version=MODULE_VERSION,
|
||||
dependencies=(),
|
||||
optional_dependencies=(
|
||||
"access",
|
||||
"audit",
|
||||
"connectors",
|
||||
"files",
|
||||
"notifications",
|
||||
"policy",
|
||||
"reporting",
|
||||
"risk_compliance",
|
||||
"workflow",
|
||||
),
|
||||
provides_interfaces=(
|
||||
ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name="dataflow.dataset_output", version=MODULE_VERSION),
|
||||
),
|
||||
permissions=PERMISSIONS,
|
||||
role_templates=ROLE_TEMPLATES,
|
||||
nav_items=(
|
||||
NavItem(
|
||||
path="/dataflow",
|
||||
label="Dataflow",
|
||||
icon="waypoints",
|
||||
required_any=(READ_SCOPE, ADMIN_SCOPE),
|
||||
order=72,
|
||||
),
|
||||
),
|
||||
frontend=FrontendModule(
|
||||
module_id=MODULE_ID,
|
||||
package_name="@govoplan/dataflow-webui",
|
||||
nav_items=(
|
||||
NavItem(
|
||||
path="/dataflow",
|
||||
label="Dataflow",
|
||||
icon="waypoints",
|
||||
required_any=(READ_SCOPE, ADMIN_SCOPE),
|
||||
order=72,
|
||||
),
|
||||
),
|
||||
),
|
||||
route_factory=_dataflow_router,
|
||||
tenant_summary_providers=(_tenant_summary,),
|
||||
migration_spec=MigrationSpec(
|
||||
module_id=MODULE_ID,
|
||||
metadata=Base.metadata,
|
||||
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||
retirement_supported=True,
|
||||
retirement_provider=drop_table_retirement_provider(
|
||||
dataflow_models.DataflowRun,
|
||||
dataflow_models.DataflowPipelineRevision,
|
||||
dataflow_models.DataflowPipeline,
|
||||
label="Dataflow",
|
||||
),
|
||||
retirement_notes=(
|
||||
"Destructive retirement drops Dataflow definitions, revisions, and run evidence "
|
||||
"after the installer captures a database snapshot."
|
||||
),
|
||||
),
|
||||
uninstall_guard_providers=(
|
||||
persistent_table_uninstall_guard(
|
||||
dataflow_models.DataflowPipeline,
|
||||
dataflow_models.DataflowPipelineRevision,
|
||||
dataflow_models.DataflowRun,
|
||||
label="Dataflow",
|
||||
),
|
||||
),
|
||||
documentation=DOCUMENTATION,
|
||||
)
|
||||
|
||||
|
||||
def get_manifest() -> ModuleManifest:
|
||||
return manifest
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ADMIN_SCOPE",
|
||||
"MODULE_ID",
|
||||
"MODULE_NAME",
|
||||
"MODULE_VERSION",
|
||||
"READ_SCOPE",
|
||||
"RUN_SCOPE",
|
||||
"WRITE_SCOPE",
|
||||
"get_manifest",
|
||||
"manifest",
|
||||
]
|
||||
Reference in New Issue
Block a user