GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed reusable pipelines and triggers

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_manifest.py
View File

@@ -2,7 +2,10 @@ from __future__ import annotations
import unittest
from govoplan_core.core.dataflows import CAPABILITY_DATAFLOW_RUN_LIFECYCLE
from govoplan_core.core.dataflows import (
CAPABILITY_DATAFLOW_RUN_LIFECYCLE,
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
)
from govoplan_core.core.modules import ModuleManifest
from govoplan_dataflow.backend.manifest import get_manifest
@@ -24,12 +27,17 @@ class DataflowManifestTests(unittest.TestCase):
CAPABILITY_DATAFLOW_RUN_LIFECYCLE,
manifest.capability_factories,
)
self.assertIn(
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
manifest.capability_factories,
)
self.assertEqual(
{
"dataflow.pipeline_catalog",
"dataflow.pipeline_preview",
"dataflow.run_lifecycle",
"dataflow.dataset_output",
"dataflow.trigger_dispatcher",
},
{item.name for item in manifest.provides_interfaces},
)

4
tests/test_migrations.py
View File

@@ -24,7 +24,7 @@ class DataflowMigrationTests(unittest.TestCase):
try:
with engine.connect() as connection:
self.assertIn(
"b8e3c6a1f4d9",
"e2f4a8c9d1b7",
set(MigrationContext.configure(connection).get_current_heads()),
)
self.assertEqual(
@@ -32,6 +32,8 @@ class DataflowMigrationTests(unittest.TestCase):
"dataflow_pipelines",
"dataflow_pipeline_revisions",
"dataflow_runs",
"dataflow_triggers",
"dataflow_trigger_deliveries",
},
{
name

3
tests/test_service.py
View File

@@ -173,6 +173,7 @@ class DataflowServiceTests(unittest.TestCase):
payload=PipelineCreateRequest(
name="Monthly comparison",
description="First governed pipeline",
status="active",
graph=sample_graph(),
editor_mode="graph",
),
@@ -277,6 +278,7 @@ class DataflowServiceTests(unittest.TestCase):
tenant_id="tenant-1",
actor_id="user-1",
payload=PipelinePreviewRequest(pipeline_id=pipeline.id, row_limit=1),
principal=principal(),
)
self.session.commit()
@@ -316,6 +318,7 @@ class DataflowServiceTests(unittest.TestCase):
tenant_id="tenant-1",
actor_id="user-1",
payload=PipelinePreviewRequest(pipeline_id=pipeline.id),
principal=principal(),
)
self.session.commit()

527
tests/test_triggers.py Normal file
View File

@@ -0,0 +1,527 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import unittest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.automation import AutomationPrincipalResolution
from govoplan_core.core.policy import PolicyDecision
from govoplan_core.db.base import Base, utcnow
from govoplan_dataflow.backend.db.models import (
DataflowPipeline,
DataflowPipelineRevision,
DataflowRun,
DataflowTrigger,
DataflowTriggerDelivery,
)
from govoplan_dataflow.backend.schemas import (
DataflowEventDeliveryRequest,
DataflowTriggerCreateRequest,
DataflowTriggerEvent,
DataflowTriggerSchedule,
PipelineCreateRequest,
PipelineDeriveRequest,
PipelineUpdateRequest,
)
from govoplan_dataflow.backend.service import (
DataflowConflictError,
create_pipeline,
derive_pipeline,
start_pipeline_run,
update_pipeline,
)
from govoplan_core.core.dataflows import DataflowRunRequest
from govoplan_dataflow.backend.triggers import (
create_trigger,
dispatch_due_triggers,
ingest_event_delivery,
)
POLICY_CAPABILITY = "policy.definitionGovernance"
AUTOMATION_CAPABILITY = "auth.automationPrincipalProvider"
def sample_graph():
from test_service import sample_graph as build_graph
return build_graph()
def principal() -> ApiPrincipal:
return ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="membership-1",
tenant_id="tenant-1",
scopes=frozenset(
{
"dataflow:pipeline:run",
"dataflow:trigger:read",
"dataflow:trigger:write",
}
),
),
account=object(),
user=object(),
)
class AllowDefinitionPolicy:
def resolve_definition_action(self, *, request):
allowed = not (
request.definition_kind == "template"
and request.action in {"run", "automate"}
)
if request.action == "automate":
allowed = allowed and request.allow_automation
if request.action == "run":
allowed = (
allowed
and request.allow_run
and request.status == "active"
)
if request.action in {"reuse", "derive"}:
allowed = allowed and request.allow_reuse
return PolicyDecision(
allowed=allowed,
reason=None if allowed else "Definition action denied.",
)
class CurrentPrincipalProvider:
def __init__(self, actor: ApiPrincipal, *, allowed: bool = True) -> None:
self.actor = actor
self.allowed = allowed
def resolve_automation_principal(self, _session, *, request):
return AutomationPrincipalResolution(
allowed=self.allowed,
principal=self.actor if self.allowed else None,
reason=None if self.allowed else "Owner authorization revoked.",
granted_scopes=tuple(request.grant_scopes) if self.allowed else (),
missing_scopes=() if self.allowed else tuple(request.grant_scopes),
provenance={"status": "current" if self.allowed else "revoked"},
)
class Registry:
def __init__(
self,
actor: ApiPrincipal,
*,
automation_allowed: bool = True,
) -> None:
self.capabilities = {
POLICY_CAPABILITY: AllowDefinitionPolicy(),
AUTOMATION_CAPABILITY: CurrentPrincipalProvider(
actor,
allowed=automation_allowed,
),
}
def has_capability(self, name: str) -> bool:
return name in self.capabilities
def capability(self, name: str):
return self.capabilities[name]
class DataflowTriggerTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(
self.engine,
tables=[
DataflowPipeline.__table__,
DataflowPipelineRevision.__table__,
DataflowRun.__table__,
DataflowTrigger.__table__,
DataflowTriggerDelivery.__table__,
],
)
self.Session = sessionmaker(bind=self.engine)
self.session: Session = self.Session()
self.actor = principal()
self.registry = Registry(self.actor)
self.pipeline = create_pipeline(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=PipelineCreateRequest(
name="Automated comparison",
status="active",
allow_automation=True,
graph=sample_graph(),
),
)
self.session.commit()
def tearDown(self) -> None:
self.session.close()
Base.metadata.drop_all(
self.engine,
tables=[
DataflowTriggerDelivery.__table__,
DataflowTrigger.__table__,
DataflowRun.__table__,
DataflowPipelineRevision.__table__,
DataflowPipeline.__table__,
],
)
self.engine.dispose()
def _event_trigger(self) -> DataflowTrigger:
trigger = create_trigger(
self.session,
tenant_id="tenant-1",
pipeline_id=self.pipeline.id,
actor_id="account-1",
principal=self.actor,
registry=self.registry,
payload=DataflowTriggerCreateRequest(
name="Monthly file arrived",
kind="event",
enabled=True,
event=DataflowTriggerEvent(
event_type="files.upload.completed",
module_id="files",
filters={"folder_id": "incoming"},
),
),
)
self.session.commit()
return trigger
def test_interval_delivery_runs_pinned_revision_with_provenance(self) -> None:
trigger = create_trigger(
self.session,
tenant_id="tenant-1",
pipeline_id=self.pipeline.id,
actor_id="account-1",
principal=self.actor,
registry=self.registry,
payload=DataflowTriggerCreateRequest(
name="Hourly",
kind="interval",
enabled=True,
schedule=DataflowTriggerSchedule(
interval_seconds=3600,
timezone="Europe/Berlin",
),
),
)
now = utcnow()
trigger.next_fire_at = now - timedelta(minutes=1)
self.session.commit()
result = dispatch_due_triggers(
self.session,
registry=self.registry,
now=now,
)
self.session.commit()
delivery = self.session.scalar(
select(DataflowTriggerDelivery).where(
DataflowTriggerDelivery.trigger_id == trigger.id
)
)
run = self.session.get(DataflowRun, delivery.run_id)
self.assertEqual(1, result.queued)
self.assertEqual(1, result.succeeded)
self.assertEqual("succeeded", delivery.status)
self.assertEqual("schedule", run.invocation_kind)
self.assertEqual(trigger.id, run.trigger_id)
self.assertEqual(delivery.id, run.trigger_delivery_id)
self.assertEqual("current", delivery.authorization_provenance["status"])
def test_skip_catch_up_records_missed_interval_without_running(self) -> None:
trigger = create_trigger(
self.session,
tenant_id="tenant-1",
pipeline_id=self.pipeline.id,
actor_id="account-1",
principal=self.actor,
registry=self.registry,
payload=DataflowTriggerCreateRequest(
name="Skip stale intervals",
kind="interval",
enabled=True,
catch_up_policy="skip",
schedule=DataflowTriggerSchedule(
anchor_at=datetime(
2000,
1,
1,
tzinfo=timezone.utc,
),
interval_seconds=60,
),
),
)
now = utcnow()
trigger.next_fire_at = now - timedelta(minutes=3)
self.session.commit()
result = dispatch_due_triggers(
self.session,
registry=self.registry,
now=now,
)
self.session.commit()
delivery = self.session.scalar(
select(DataflowTriggerDelivery).where(
DataflowTriggerDelivery.trigger_id == trigger.id
)
)
self.assertEqual(0, result.queued)
self.assertEqual(1, result.skipped)
self.assertEqual("skipped", delivery.status)
self.assertIsNone(delivery.run_id)
next_fire_at = trigger.next_fire_at
self.assertIsNotNone(next_fire_at)
if next_fire_at.tzinfo is None:
next_fire_at = next_fire_at.replace(tzinfo=timezone.utc)
self.assertGreater(next_fire_at, now)
def test_disabled_trigger_can_be_prepared_for_a_draft(self) -> None:
self.pipeline.status = "draft"
self.session.flush()
trigger = create_trigger(
self.session,
tenant_id="tenant-1",
pipeline_id=self.pipeline.id,
actor_id="account-1",
principal=self.actor,
registry=self.registry,
payload=DataflowTriggerCreateRequest(
name="Prepared event",
kind="event",
enabled=False,
event=DataflowTriggerEvent(
event_type="files.upload.completed",
),
),
)
self.assertEqual("disabled", trigger.status)
self.assertIsNone(trigger.next_fire_at)
def test_event_ingress_is_filtered_and_idempotent(self) -> None:
trigger = self._event_trigger()
event = DataflowEventDeliveryRequest(
event_id="event-1",
type="files.upload.completed",
module_id="files",
payload={"folder_id": "incoming", "file_id": "file-1"},
occurred_at=utcnow(),
)
self.assertEqual(
1,
ingest_event_delivery(
self.session,
tenant_id="tenant-1",
event=event,
),
)
self.assertEqual(
0,
ingest_event_delivery(
self.session,
tenant_id="tenant-1",
event=event,
),
)
result = dispatch_due_triggers(
self.session,
registry=self.registry,
)
self.session.commit()
delivery = self.session.scalar(
select(DataflowTriggerDelivery).where(
DataflowTriggerDelivery.trigger_id == trigger.id
)
)
run = self.session.get(DataflowRun, delivery.run_id)
self.assertEqual(1, result.succeeded)
self.assertEqual("event", run.invocation_kind)
self.assertEqual("event-1", run.request_["invocation"]["event_id"])
def test_revoked_owner_blocks_delivery_before_execution(self) -> None:
trigger = self._event_trigger()
ingest_event_delivery(
self.session,
tenant_id="tenant-1",
event=DataflowEventDeliveryRequest(
event_id="event-revoked",
type="files.upload.completed",
module_id="files",
payload={"folder_id": "incoming"},
occurred_at=utcnow(),
),
)
result = dispatch_due_triggers(
self.session,
registry=Registry(self.actor, automation_allowed=False),
)
self.session.commit()
delivery = self.session.scalar(
select(DataflowTriggerDelivery).where(
DataflowTriggerDelivery.trigger_id == trigger.id
)
)
self.assertEqual(1, result.blocked)
self.assertEqual("blocked", delivery.status)
self.assertIn("revoked", delivery.error)
self.assertIsNone(delivery.run_id)
def test_restricted_event_requires_durable_bridge(self) -> None:
self._event_trigger()
with self.assertRaises(DataflowConflictError):
ingest_event_delivery(
self.session,
tenant_id="tenant-1",
event=DataflowEventDeliveryRequest(
event_id="event-secret",
type="files.upload.completed",
module_id="files",
payload={"folder_id": "incoming"},
occurred_at=utcnow(),
classification="restricted",
),
)
def test_template_derivation_pins_source_and_template_cannot_run(self) -> None:
template = create_pipeline(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=PipelineCreateRequest(
name="System import template",
graph=sample_graph(),
scope_type="system",
definition_kind="template",
inherit_to_lower_scopes=True,
allow_reuse=True,
),
)
derived = derive_pipeline(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
principal=self.actor,
registry=self.registry,
source_pipeline_id=template.id,
payload=PipelineDeriveRequest(
name="Tenant monthly import",
definition_kind="flow",
allow_run=True,
),
)
self.session.commit()
self.assertEqual(template.id, derived.derived_from_pipeline_id)
self.assertEqual(1, derived.derived_from_revision)
self.assertEqual(
template.revisions[0].content_hash,
derived.derived_from_hash,
)
self.assertEqual(
"system",
derived.derivation_provenance["source_scope"]["scope_type"],
)
with self.assertRaises(DataflowConflictError):
start_pipeline_run(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
principal=self.actor,
registry=self.registry,
request=DataflowRunRequest(
pipeline_ref=f"pipeline:{template.id}",
revision=1,
idempotency_key="template-must-not-run",
),
)
def test_derived_limits_cannot_be_broadened_transitively(self) -> None:
template = create_pipeline(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
payload=PipelineCreateRequest(
name="Restricted template",
graph=sample_graph(),
definition_kind="template",
allow_reuse=True,
allow_automation=False,
inherit_to_lower_scopes=False,
),
)
derived = derive_pipeline(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
principal=self.actor,
registry=self.registry,
source_pipeline_id=template.id,
payload=PipelineDeriveRequest(
name="Tenant flow",
allow_reuse=True,
allow_automation=True,
inherit_to_lower_scopes=True,
),
)
self.assertFalse(derived.allow_automation)
self.assertFalse(derived.inherit_to_lower_scopes)
update_pipeline(
self.session,
tenant_id="tenant-1",
pipeline_id=derived.id,
actor_id="account-1",
payload=PipelineUpdateRequest(
name=derived.name,
graph=sample_graph(),
status="draft",
expected_revision=1,
allow_reuse=True,
allow_automation=True,
inherit_to_lower_scopes=True,
),
)
grandchild = derive_pipeline(
self.session,
tenant_id="tenant-1",
actor_id="account-1",
principal=self.actor,
registry=self.registry,
source_pipeline_id=derived.id,
payload=PipelineDeriveRequest(
name="User flow",
scope_type="user",
scope_id="membership-1",
allow_automation=True,
inherit_to_lower_scopes=True,
),
)
self.assertFalse(derived.allow_automation)
self.assertFalse(derived.inherit_to_lower_scopes)
self.assertFalse(grandchild.allow_automation)
self.assertFalse(grandchild.inherit_to_lower_scopes)
if __name__ == "__main__":
unittest.main()