GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed reusable pipelines and triggers

Browse Source
This commit is contained in:
zemion
2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_dataflow/backend/manifest.py
+84 -4
View File
@@ -6,7 +6,13 @@ from govoplan_core.core.module_guards import (
drop_table_retirement_provider,
persistent_table_uninstall_guard,
)
from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_RUN_LIFECYCLE
from govoplan_core.core.access import (
CAPABILITY_AUTH_AUTOMATION_PRINCIPAL_PROVIDER,
)
from govoplan_core.core.dataflows import (
CAPABILITY_DATAFLOW_RUN_LIFECYCLE,
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
)
from govoplan_core.core.modules import (
DocumentationTopic,
FrontendModule,
@@ -24,6 +30,9 @@ from govoplan_core.core.datasources import (
CAPABILITY_DATASOURCE_LIFECYCLE,
CAPABILITY_DATASOURCE_PUBLICATION,
)
from govoplan_core.core.policy import (
CAPABILITY_POLICY_DEFINITION_GOVERNANCE,
)
from govoplan_core.db.base import Base
from govoplan_dataflow.backend.db import models as dataflow_models
@@ -36,6 +45,9 @@ READ_SCOPE = "dataflow:pipeline:read"
WRITE_SCOPE = "dataflow:pipeline:write"
RUN_SCOPE = "dataflow:pipeline:run"
ADMIN_SCOPE = "dataflow:pipeline:admin"
TRIGGER_READ_SCOPE = "dataflow:trigger:read"
TRIGGER_WRITE_SCOPE = "dataflow:trigger:write"
TRIGGER_DISPATCH_SCOPE = "dataflow:trigger:dispatch"
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
@@ -73,6 +85,21 @@ PERMISSIONS = (
"Administer Dataflow",
"Manage every tenant pipeline and future execution, retention, and publication policies.",
),
_permission(
TRIGGER_READ_SCOPE,
"View Dataflow triggers",
"Inspect schedule and event trigger configuration, status, provenance, and deliveries.",
),
_permission(
TRIGGER_WRITE_SCOPE,
"Manage Dataflow triggers",
"Create, edit, enable, disable, and remove governed Dataflow triggers.",
),
_permission(
TRIGGER_DISPATCH_SCOPE,
"Dispatch Dataflow triggers",
"Ingest trusted events and dispatch queued automated Dataflow runs.",
),
)
ROLE_TEMPLATES = (
@@ -80,19 +107,25 @@ ROLE_TEMPLATES = (
slug="dataflow_designer",
name="Dataflow designer",
description="Design, validate, and preview governed data pipelines.",
permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE),
permissions=(
READ_SCOPE,
WRITE_SCOPE,
RUN_SCOPE,
TRIGGER_READ_SCOPE,
TRIGGER_WRITE_SCOPE,
),
),
RoleTemplate(
slug="dataflow_operator",
name="Dataflow operator",
description="Inspect and run approved data pipelines without changing their definitions.",
permissions=(READ_SCOPE, RUN_SCOPE),
permissions=(READ_SCOPE, RUN_SCOPE, TRIGGER_READ_SCOPE),
),
RoleTemplate(
slug="dataflow_viewer",
name="Dataflow viewer",
description="Inspect pipeline definitions, revisions, diagnostics, and run summaries.",
permissions=(READ_SCOPE,),
permissions=(READ_SCOPE, TRIGGER_READ_SCOPE),
),
)
@@ -151,6 +184,14 @@ def _run_provider(context: ModuleContext):
return SqlDataflowRunLifecycleProvider(registry=context.registry)
def _trigger_provider(context: ModuleContext):
from govoplan_dataflow.backend.triggers import (
SqlDataflowTriggerDispatcher,
)
return SqlDataflowTriggerDispatcher(registry=context.registry)
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
return {
"dataflow_pipelines": (
@@ -166,6 +207,19 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
.filter(dataflow_models.DataflowRun.tenant_id == tenant_id)
.count()
),
"dataflow_triggers": (
session.query(dataflow_models.DataflowTrigger)
.filter(dataflow_models.DataflowTrigger.tenant_id == tenant_id)
.count()
),
"dataflow_trigger_deliveries": (
session.query(dataflow_models.DataflowTriggerDelivery)
.filter(
dataflow_models.DataflowTriggerDelivery.tenant_id
== tenant_id
)
.count()
),
}
@@ -189,12 +243,18 @@ manifest = ModuleManifest(
CAPABILITY_DATASOURCE_CATALOGUE,
CAPABILITY_DATASOURCE_LIFECYCLE,
CAPABILITY_DATASOURCE_PUBLICATION,
CAPABILITY_AUTH_AUTOMATION_PRINCIPAL_PROVIDER,
CAPABILITY_POLICY_DEFINITION_GOVERNANCE,
),
provides_interfaces=(
ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION),
ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION),
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
ModuleInterfaceProvider(name="dataflow.dataset_output", version=MODULE_VERSION),
ModuleInterfaceProvider(
name="dataflow.trigger_dispatcher",
version=MODULE_VERSION,
),
),
requires_interfaces=(
ModuleInterfaceRequirement(
@@ -203,6 +263,18 @@ manifest = ModuleManifest(
version_max_exclusive="1.0.0",
optional=True,
),
ModuleInterfaceRequirement(
name="policy.definition_governance",
version_min="0.1.0",
version_max_exclusive="1.0.0",
optional=True,
),
ModuleInterfaceRequirement(
name="auth.automation_principal",
version_min="0.1.0",
version_max_exclusive="1.0.0",
optional=True,
),
ModuleInterfaceRequirement(
name="datasources.lifecycle",
version_min="0.1.0",
@@ -243,6 +315,7 @@ manifest = ModuleManifest(
route_factory=_dataflow_router,
capability_factories={
CAPABILITY_DATAFLOW_RUN_LIFECYCLE: _run_provider,
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER: _trigger_provider,
},
tenant_summary_providers=(_tenant_summary,),
migration_spec=MigrationSpec(
@@ -251,6 +324,8 @@ manifest = ModuleManifest(
script_location=str(Path(__file__).with_name("migrations") / "versions"),
retirement_supported=True,
retirement_provider=drop_table_retirement_provider(
dataflow_models.DataflowTriggerDelivery,
dataflow_models.DataflowTrigger,
dataflow_models.DataflowRun,
dataflow_models.DataflowPipelineRevision,
dataflow_models.DataflowPipeline,
@@ -266,6 +341,8 @@ manifest = ModuleManifest(
dataflow_models.DataflowPipeline,
dataflow_models.DataflowPipelineRevision,
dataflow_models.DataflowRun,
dataflow_models.DataflowTrigger,
dataflow_models.DataflowTriggerDelivery,
label="Dataflow",
),
),
@@ -284,6 +361,9 @@ __all__ = [
"MODULE_VERSION",
"READ_SCOPE",
"RUN_SCOPE",
"TRIGGER_DISPATCH_SCOPE",
"TRIGGER_READ_SCOPE",
"TRIGGER_WRITE_SCOPE",
"WRITE_SCOPE",
"get_manifest",
"manifest",