GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed Dataflow audience outputs

Browse Source
This commit is contained in:
zemion
2026-07-31 21:00:02 +02:00
parent 8724591bf9
commit cef2ba9482
12 changed files with 1226 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_dataset_output.py
+150
View File
@@ -0,0 +1,150 @@
from __future__ import annotations
import unittest
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.access import PrincipalRef
from govoplan_core.core.dataflows import DataflowDatasetRequest, DataflowRunConflictError
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, reset_database
from govoplan_dataflow.backend.dataset_output import SqlDataflowDatasetOutputProvider
from govoplan_dataflow.backend.db.models import DataflowPipeline, DataflowPipelineRevision
from govoplan_dataflow.backend.schemas import PipelineGraph
from govoplan_dataflow.backend.service import definition_hash
def principal(tenant_id: str = "tenant-1") -> ApiPrincipal:
return ApiPrincipal(
principal=PrincipalRef(
account_id="account-1",
membership_id="membership-1",
tenant_id=tenant_id,
scopes=frozenset({"dataflow:pipeline:read", "dataflow:pipeline:run"}),
),
account=object(),
user=object(),
)
class DataflowDatasetOutputTests(unittest.TestCase):
def setUp(self) -> None:
self.database = configure_database("sqlite:///:memory:")
Base.metadata.create_all(
self.database.engine,
tables=[DataflowPipeline.__table__, DataflowPipelineRevision.__table__],
)
def tearDown(self) -> None:
reset_database(dispose=True)
def test_pinned_revision_executes_through_typed_registry(self) -> None:
graph = PipelineGraph.model_validate(
{
"nodes": [
{
"id": "source",
"type": "source.inline",
"label": "Source",
"position": {"x": 0, "y": 0},
"config": {
"source_name": "audience",
"rows": [
{"recipient_key": "one", "email": "one@example.test"},
{"recipient_key": "two", "email": "two@example.test"},
]
},
},
{
"id": "output",
"type": "output",
"label": "Output",
"position": {"x": 200, "y": 0},
"config": {},
},
],
"edges": [{"id": "edge-1", "source": "source", "target": "output"}],
}
)
content_hash = definition_hash(graph, None)
with self.database.session() as session:
pipeline = DataflowPipeline(
id="pipeline-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
definition_kind="flow",
name="Audience",
status="active",
current_revision=1,
allow_run=True,
metadata_={},
)
revision = DataflowPipelineRevision(
id="revision-1",
tenant_id="tenant-1",
pipeline_id=pipeline.id,
revision=1,
graph=graph.model_dump(mode="json"),
sql_text=None,
editor_mode="graph",
content_hash=content_hash,
)
session.add_all((pipeline, revision))
session.flush()
provider = SqlDataflowDatasetOutputProvider()
descriptors = provider.list_outputs(session, principal())
result = provider.read_output(
session,
principal(),
request=DataflowDatasetRequest(
pipeline_ref=pipeline.id,
revision=1,
expected_definition_hash=content_hash,
),
)
self.assertEqual(1, len(descriptors))
self.assertEqual(2, result.total_rows)
self.assertEqual("one", result.rows[0]["recipient_key"])
self.assertEqual(64, len(result.output_hash))
replay = provider.read_output(
session,
principal(),
request=DataflowDatasetRequest(
pipeline_ref=pipeline.id,
revision=1,
expected_definition_hash=content_hash,
expected_source_fingerprints=result.source_fingerprints,
),
)
self.assertEqual(result.output_hash, replay.output_hash)
with self.assertRaises(DataflowRunConflictError):
provider.read_output(
session,
principal(),
request=DataflowDatasetRequest(
pipeline_ref=pipeline.id,
revision=1,
expected_definition_hash="wrong",
),
)
with self.assertRaises(DataflowRunConflictError):
provider.read_output(
session,
principal(),
request=DataflowDatasetRequest(
pipeline_ref=pipeline.id,
revision=1,
expected_definition_hash=content_hash,
expected_source_fingerprints=(
{"node_id": "source", "fingerprint": "changed"},
),
),
)
if __name__ == "__main__":
unittest.main()
tests/test_golden_flows.py
+147 -5
View File
@@ -1,12 +1,19 @@
from __future__ import annotations
import json
import hashlib
import unittest
from pathlib import Path
from govoplan_dataflow.backend.executor import execute_preview
from govoplan_dataflow.backend.graph import validate_graph
from govoplan_dataflow.backend.executor import (
PipelineExecutionError,
ResolvedSource,
execute_preview,
)
from govoplan_dataflow.backend.expressions import ExpressionError, parse_expression
from govoplan_dataflow.backend.graph import definition_hash, validate_graph
from govoplan_dataflow.backend.schemas import PipelineGraph
from govoplan_dataflow.backend.subflows import substitute_parameters
FIXTURES = Path(__file__).parents[1] / "fixtures" / "golden"
@@ -21,13 +28,18 @@ class GoldenFlowTests(unittest.TestCase):
for directory in directories:
with self.subTest(flow=directory.name):
manifest = _json(directory / "manifest.json")
graph_payload = _json(directory / str(manifest["graph"]))
raw_graph_payload = _json(directory / str(manifest["graph"]))
graph_payload = json.loads(json.dumps(raw_graph_payload))
for item in graph_payload["nodes"]:
fixture = item.get("config", {}).get("fixture")
if fixture:
if fixture and item["type"] == "source.inline":
item["config"]["rows"] = _json(
directory / "inputs" / fixture
)
graph_payload = substitute_parameters(
graph_payload,
dict(manifest.get("parameters") or {}),
)
graph = PipelineGraph.model_validate(graph_payload)
diagnostics = validate_graph(graph)
self.assertEqual(
@@ -39,17 +51,147 @@ class GoldenFlowTests(unittest.TestCase):
],
)
result = execute_preview(graph, row_limit=500)
resolver = _source_resolver(directory, manifest)
result = execute_preview(
graph,
row_limit=500,
source_resolver=resolver,
)
self.assertEqual(
_json(directory / str(manifest["expected_output"])),
result.rows,
)
if "expected_definition_hash" in manifest:
self.assertEqual(
manifest["expected_definition_hash"],
definition_hash(PipelineGraph.model_validate(raw_graph_payload)),
)
if "expected_execution_graph_hash" in manifest:
self.assertEqual(
manifest["expected_execution_graph_hash"],
definition_hash(graph),
)
if "expected_output_hash" in manifest:
self.assertEqual(
manifest["expected_output_hash"],
_rows_hash(result.rows),
)
if "expected_source_fingerprints" in manifest:
self.assertEqual(
manifest["expected_source_fingerprints"],
result.source_fingerprints,
)
full_result = execute_preview(
graph,
row_limit=5_000,
source_resolver=resolver,
)
self.assertEqual(result.rows, full_result.rows)
self.assertFalse(full_result.truncated)
def test_adrema_fixture_rejects_untrusted_sources_and_unsafe_expression(self) -> None:
directory = FIXTURES / "adrema-audience-selection"
manifest = _json(directory / "manifest.json")
graph = PipelineGraph.model_validate(
substitute_parameters(
_json(directory / "graph.json"),
dict(manifest["parameters"]),
)
)
drifted_manifest = {
**manifest,
"sources": {
**manifest["sources"],
"snapshot:directory-2026-01": {
**manifest["sources"]["snapshot:directory-2026-01"],
"fingerprint": "sha256:changed-source",
},
},
}
with self.assertRaisesRegex(PipelineExecutionError, "fingerprint"):
execute_preview(
graph,
row_limit=500,
source_resolver=_source_resolver(directory, drifted_manifest),
)
cross_tenant_manifest = {
**manifest,
"sources": {
**manifest["sources"],
"snapshot:directory-2026-01": {
**manifest["sources"]["snapshot:directory-2026-01"],
"tenant_id": "tenant-other",
},
},
}
with self.assertRaisesRegex(PipelineExecutionError, "tenant"):
execute_preview(
graph,
row_limit=500,
source_resolver=_source_resolver(directory, cross_tenant_manifest),
)
unknown_source_graph = graph.model_copy(deep=True)
unknown_source_graph.nodes[0].config["source_ref"] = "connector:untrusted"
with self.assertRaisesRegex(PipelineExecutionError, "pinned provider set"):
execute_preview(
unknown_source_graph,
row_limit=500,
source_resolver=_source_resolver(directory, manifest),
)
with self.assertRaises(ExpressionError):
parse_expression("read_csv('/tmp/private.csv')")
def _json(path: Path):
return json.loads(path.read_text(encoding="utf-8"))
def _source_resolver(directory: Path, manifest: dict):
sources = dict(manifest.get("sources") or {})
tenant_id = str(manifest.get("tenant_id") or "")
def resolve(node, limit: int) -> ResolvedSource:
source_ref = str(node.config.get("source_ref") or "")
if source_ref not in sources:
raise ValueError(f"Fixture source {source_ref!r} is outside the pinned provider set.")
source = sources[source_ref]
if str(source.get("tenant_id") or "") != tenant_id:
raise ValueError(
f"Fixture source {source_ref!r} belongs to a different tenant."
)
fingerprint = str(source["fingerprint"])
expected = str(node.config.get("expected_fingerprint") or "")
if expected and expected != fingerprint:
raise ValueError(
f"Fixture source fingerprint changed for {source_ref!r}."
)
rows = _json(directory / str(source["fixture"]))
bounded = rows[:limit]
return ResolvedSource(
rows=tuple(dict(item) for item in bounded),
source_ref=source_ref,
provider=str(source["provider"]),
fingerprint=fingerprint,
total_rows=len(rows),
truncated=len(bounded) < len(rows),
)
return resolve
def _rows_hash(rows: object) -> str:
return hashlib.sha256(
json.dumps(
rows,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
).hexdigest()
if __name__ == "__main__":
unittest.main()
tests/test_manifest.py
+5
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import unittest
from govoplan_core.core.dataflows import (
CAPABILITY_DATAFLOW_DATASET_OUTPUT,
CAPABILITY_DATAFLOW_RUN_LIFECYCLE,
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
)
@@ -31,6 +32,10 @@ class DataflowManifestTests(unittest.TestCase):
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
manifest.capability_factories,
)
self.assertIn(
CAPABILITY_DATAFLOW_DATASET_OUTPUT,
manifest.capability_factories,
)
self.assertEqual(
{
"dataflow.pipeline_catalog",