GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: consume governed datasources and shared graphs

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-28 12:44:46 +02:00
parent dee8380631
commit 6ca3058021
15 changed files with 248 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
README.md
View File

@@ -10,8 +10,10 @@ the previewed row contents.
- **Dataflow:** pipeline graphs, revisions, validation, constrained SQL, - **Dataflow:** pipeline graphs, revisions, validation, constrained SQL,
previews/runs, diagnostics, and lineage. previews/runs, diagnostics, and lineage.
- **Connectors:** source connections, credentials, discovery, health, and - **Datasources:** governed source identity, staging, materializations, frozen
bounded source access. states, and bounded source access.
- **Connectors:** external acquisition, credentials, discovery, and provider
health.
- **Reporting:** governed datasets, analytical views, dashboards, and exports. - **Reporting:** governed datasets, analytical views, dashboards, and exports.
- **Workflow:** orchestration, resumability, approvals, and module handoffs. - **Workflow:** orchestration, resumability, approvals, and module handoffs.
- **Risk Compliance:** sanctions matching policy, review, dispositions, and - **Risk Compliance:** sanctions matching policy, review, dispositions, and
@@ -24,7 +26,7 @@ nodes by purpose:
| Group | Nodes | | Group | Nodes |
| --- | --- | | --- | --- |
| Load | Inline data, connector source | | Load | Inline data, datasource |
| Combine | Append rows, join tables | | Combine | Append rows, join tables |
| Filter | Filter rows, remove duplicates | | Filter | Filter rows, remove duplicates |
| Transform | Select columns, derive column, aggregate, sort, limit | | Transform | Select columns, derive column, aggregate, sort, limit |
@@ -34,10 +36,11 @@ Join nodes have explicit left and right ports. Append nodes accept two or more
inputs. Derived columns use a constrained operation catalogue rather than inputs. Derived columns use a constrained operation catalogue rather than
arbitrary code. arbitrary code.
Connector sources are resolved through the versioned Core capability Governed inputs are resolved through the versioned Core capability
`connectors.tabular_sources`; Dataflow does not import Connectors or store its `datasources.catalogue`; Dataflow imports neither Datasources nor Connectors and
credentials. Source references pin an expected fingerprint so schema/content stores only opaque datasource references plus expected fingerprints. A source
drift fails visibly instead of silently changing a run. node can request the current, live, or latest frozen state. Fingerprint drift
fails visibly instead of silently changing a run.
## SQL And Preview Safety ## SQL And Preview Safety

4
src/govoplan_dataflow/backend/executor.py
View File

@@ -137,7 +137,7 @@ def execute_preview(
elif node.type == "source.reference": elif node.type == "source.reference":
if source_resolver is None: if source_resolver is None:
raise PipelineExecutionError( raise PipelineExecutionError(
"Connector-backed preview requires the Connectors tabular-source capability.", "Datasource-backed preview requires the Datasources catalogue capability.",
node_id=node.id, node_id=node.id,
) )
resolved = source_resolver(node, MAX_SOURCE_ROWS) resolved = source_resolver(node, MAX_SOURCE_ROWS)
@@ -148,7 +148,7 @@ def execute_preview(
"node_id": node.id, "node_id": node.id,
"source_ref": resolved.source_ref, "source_ref": resolved.source_ref,
"source_name": node.config.get("source_name"), "source_name": node.config.get("source_name"),
"kind": "connector", "kind": "datasource",
"provider": resolved.provider, "provider": resolved.provider,
"fingerprint": resolved.fingerprint, "fingerprint": resolved.fingerprint,
"row_count": resolved.total_rows, "row_count": resolved.total_rows,

158
src/govoplan_dataflow/backend/graph.py
View File

@@ -7,7 +7,16 @@ from collections import deque
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any from typing import Any
from govoplan_dataflow.backend.node_library import NODE_TYPES, node_definition from govoplan_core.core.definition_graphs import (
DefinitionEdge,
DefinitionNode,
validate_definition_graph,
)
from govoplan_dataflow.backend.node_library import (
DATAFLOW_GRAPH_LIBRARY,
NODE_TYPES,
node_definition,
)
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphEdge, GraphNode, PipelineGraph from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphEdge, GraphNode, PipelineGraph
@@ -47,75 +56,49 @@ def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str:
def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]: def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
diagnostics: list[DataflowDiagnostic] = [] diagnostics = [
DataflowDiagnostic(
severity=item.severity,
code=item.code,
message=item.message,
node_id=item.node_id,
field=item.field,
)
for item in validate_definition_graph(
DATAFLOW_GRAPH_LIBRARY,
nodes=tuple(DefinitionNode(id=node.id, type=node.type) for node in graph.nodes),
edges=tuple(
DefinitionEdge(
id=edge.id,
source=edge.source,
target=edge.target,
source_port=edge.source_port,
target_port=edge.target_port,
)
for edge in graph.edges
),
)
]
nodes = {node.id: node for node in graph.nodes} nodes = {node.id: node for node in graph.nodes}
if len(nodes) != len(graph.nodes):
diagnostics.append(_error("graph.duplicate_node", "Node identifiers must be unique."))
edge_ids = {edge.id for edge in graph.edges}
if len(edge_ids) != len(graph.edges):
diagnostics.append(_error("graph.duplicate_edge", "Edge identifiers must be unique."))
incoming: dict[str, list[GraphEdge]] = {node_id: [] for node_id in nodes} incoming: dict[str, list[GraphEdge]] = {node_id: [] for node_id in nodes}
outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes} outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes}
for edge in graph.edges: for edge in graph.edges:
if edge.source not in nodes: if edge.source not in nodes or edge.target not in nodes or edge.source == edge.target:
diagnostics.append(
_error("edge.unknown_source", f"Edge {edge.id!r} references an unknown source node.")
)
continue
if edge.target not in nodes:
diagnostics.append(
_error("edge.unknown_target", f"Edge {edge.id!r} references an unknown target node.")
)
continue
if edge.source == edge.target:
diagnostics.append(
_error("edge.self_reference", "A node cannot connect to itself.", node_id=edge.source)
)
continue continue
source_definition = node_definition(nodes[edge.source].type) source_definition = node_definition(nodes[edge.source].type)
target_definition = node_definition(nodes[edge.target].type) target_definition = node_definition(nodes[edge.target].type)
if source_definition and edge.source_port not in { if (
port.id for port in source_definition.output_ports source_definition is None
}: or target_definition is None
diagnostics.append( or edge.source_port not in {port.id for port in source_definition.output_ports}
_error( or edge.target_port not in {port.id for port in target_definition.input_ports}
"edge.unknown_source_port", ):
f"Node {edge.source!r} has no output port {edge.source_port!r}.",
node_id=edge.source,
)
)
continue
if target_definition and edge.target_port not in {
port.id for port in target_definition.input_ports
}:
diagnostics.append(
_error(
"edge.unknown_target_port",
f"Node {edge.target!r} has no input port {edge.target_port!r}.",
node_id=edge.target,
)
)
continue continue
outgoing[edge.source].append(edge.target) outgoing[edge.source].append(edge.target)
incoming[edge.target].append(edge) incoming[edge.target].append(edge)
if not graph.nodes:
diagnostics.append(_error("graph.empty", "Add a source and an output before saving the pipeline."))
return diagnostics
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")] source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
output_nodes = [node for node in graph.nodes if node.type == "output"] output_nodes = [node for node in graph.nodes if node.type == "output"]
if not source_nodes:
diagnostics.append(
_error(
"graph.source_count",
"A pipeline needs at least one source node.",
)
)
elif len(source_nodes) > 10:
diagnostics.append(_error("graph.source_limit", "Pipelines are limited to ten sources."))
source_names = [ source_names = [
str(node.config.get("source_name", "")).strip() str(node.config.get("source_name", "")).strip()
for node in source_nodes for node in source_nodes
@@ -137,54 +120,13 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
f"{', '.join(duplicate_source_names)}.", f"{', '.join(duplicate_source_names)}.",
) )
) )
if len(output_nodes) != 1:
diagnostics.append(
_error("graph.output_count", "A pipeline must contain exactly one output node.")
)
for node in graph.nodes: for node in graph.nodes:
if node.type not in SUPPORTED_NODE_TYPES: if node.type not in SUPPORTED_NODE_TYPES:
diagnostics.append(
_error(
"node.unsupported_type",
f"Node type {node.type!r} is not supported by this executor.",
node_id=node.id,
field="type",
)
)
continue continue
definition = node_definition(node.type)
node_edges = incoming.get(node.id, [])
if definition is not None:
for port in definition.input_ports:
connections = [
edge
for edge in node_edges
if edge.target_port == port.id
]
minimum = port.minimum_connections if port.required else 0
if len(connections) < minimum:
diagnostics.append(
_error(
"node.input_required",
f"{definition.label} requires {port.label.lower()} input.",
node_id=node.id,
)
)
if not port.multiple and len(connections) > 1:
diagnostics.append(
_error(
"node.input_multiple",
f"{port.label} accepts only one connection.",
node_id=node.id,
)
)
diagnostics.extend(_validate_node_config(node)) diagnostics.extend(_validate_node_config(node))
ordered, cyclic = topological_order(graph) ordered, cyclic = topological_order(graph)
if cyclic: if not cyclic and source_nodes and len(output_nodes) == 1:
diagnostics.append(_error("graph.cycle", "Pipeline edges must form an acyclic graph."))
elif source_nodes and output_nodes:
reachable: set[str] = set() reachable: set[str] = set()
for source in source_nodes: for source in source_nodes:
reachable.update(_reachable_from(source.id, outgoing)) reachable.update(_reachable_from(source.id, outgoing))
@@ -206,7 +148,21 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
_error("graph.output_not_terminal", "The output node must be the terminal transform.") _error("graph.output_not_terminal", "The output node must be the terminal transform.")
) )
diagnostics.extend(_validate_graph_schemas(graph, ordered=ordered)) diagnostics.extend(_validate_graph_schemas(graph, ordered=ordered))
return diagnostics return _dedupe_diagnostics(diagnostics)
def _dedupe_diagnostics(
diagnostics: list[DataflowDiagnostic],
) -> list[DataflowDiagnostic]:
seen: set[tuple[str, str | None, str | None]] = set()
result: list[DataflowDiagnostic] = []
for item in diagnostics:
key = (item.code, item.node_id, item.field)
if key in seen:
continue
seen.add(key)
result.append(item)
return result
def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]: def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]:
@@ -577,7 +533,7 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
diagnostics.append( diagnostics.append(
_error( _error(
"source.reference_required", "source.reference_required",
"Choose a connector source.", "Choose a datasource.",
node_id=node.id, node_id=node.id,
field="source_ref", field="source_ref",
) )

24
src/govoplan_dataflow/backend/manifest.py
View File

@@ -18,9 +18,9 @@ from govoplan_core.core.modules import (
PermissionDefinition, PermissionDefinition,
RoleTemplate, RoleTemplate,
) )
from govoplan_core.core.tabular_sources import ( from govoplan_core.core.datasources import (
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER, CAPABILITY_DATASOURCE_CATALOGUE,
CAPABILITY_CONNECTORS_TABULAR_SOURCES, CAPABILITY_DATASOURCE_LIFECYCLE,
) )
from govoplan_core.db.base import Base from govoplan_core.db.base import Base
from govoplan_dataflow.backend.db import models as dataflow_models from govoplan_dataflow.backend.db import models as dataflow_models
@@ -101,8 +101,9 @@ DOCUMENTATION = (
summary="Versioned tabular transformations with graphical and constrained SQL editing.", summary="Versioned tabular transformations with graphical and constrained SQL editing.",
body=( body=(
"Dataflow owns canonical pipeline graphs, immutable revisions, validation, constrained " "Dataflow owns canonical pipeline graphs, immutable revisions, validation, constrained "
"SQL compilation, preview and run diagnostics, and lineage references. Connectors owns " "SQL compilation, preview and run diagnostics, and lineage references. Datasources owns "
"source connections and credentials; Reporting owns analytical presentation and exports; " "the governed catalogue and materializations, while Connectors owns external acquisition "
"and credentials; Reporting owns analytical presentation and exports; "
"Workflow owns orchestration and human handoffs; Risk Compliance owns sanctions review " "Workflow owns orchestration and human handoffs; Risk Compliance owns sanctions review "
"semantics and policy gates. User SQL is compiled into approved transforms and is never " "semantics and policy gates. User SQL is compiled into approved transforms and is never "
"passed unchecked to a backing database." "passed unchecked to a backing database."
@@ -112,6 +113,7 @@ DOCUMENTATION = (
audience=("operator", "module_admin", "power_user", "product_owner"), audience=("operator", "module_admin", "power_user", "product_owner"),
order=75, order=75,
related_modules=( related_modules=(
"datasources",
"connectors", "connectors",
"files", "files",
"reporting", "reporting",
@@ -123,7 +125,7 @@ DOCUMENTATION = (
), ),
metadata={ metadata={
"first_slice": ( "first_slice": (
"Inline and connector sources, union, join, filter, deduplication, select, " "Inline and governed datasources, union, join, filter, deduplication, select, "
"derived columns, aggregate, sort, limit, output, revisioning, and bounded preview." "derived columns, aggregate, sort, limit, output, revisioning, and bounded preview."
), ),
"sql_safety": "Constrained AST compilation only; no pass-through execution.", "sql_safety": "Constrained AST compilation only; no pass-through execution.",
@@ -167,7 +169,7 @@ manifest = ModuleManifest(
optional_dependencies=( optional_dependencies=(
"access", "access",
"audit", "audit",
"connectors", "datasources",
"files", "files",
"notifications", "notifications",
"policy", "policy",
@@ -176,8 +178,8 @@ manifest = ModuleManifest(
"workflow", "workflow",
), ),
optional_capabilities=( optional_capabilities=(
CAPABILITY_CONNECTORS_TABULAR_SOURCES, CAPABILITY_DATASOURCE_CATALOGUE,
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER, CAPABILITY_DATASOURCE_LIFECYCLE,
), ),
provides_interfaces=( provides_interfaces=(
ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION), ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION),
@@ -187,13 +189,13 @@ manifest = ModuleManifest(
), ),
requires_interfaces=( requires_interfaces=(
ModuleInterfaceRequirement( ModuleInterfaceRequirement(
name="connectors.tabular_sources", name="datasources.catalogue",
version_min="0.1.0", version_min="0.1.0",
version_max_exclusive="1.0.0", version_max_exclusive="1.0.0",
optional=True, optional=True,
), ),
ModuleInterfaceRequirement( ModuleInterfaceRequirement(
name="connectors.tabular_snapshot_writer", name="datasources.lifecycle",
version_min="0.1.0", version_min="0.1.0",
version_max_exclusive="1.0.0", version_max_exclusive="1.0.0",
optional=True, optional=True,

100
src/govoplan_dataflow/backend/node_library.py
View File

@@ -1,49 +1,30 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass, field from dataclasses import dataclass
from typing import Any, Literal from typing import Literal
from govoplan_core.core.definition_graphs import (
DefinitionConfigField,
DefinitionGraphConstraints,
DefinitionGraphLibrary,
DefinitionNodeCountConstraint,
DefinitionNodeType,
DefinitionPort,
)
NodeCategory = Literal["load", "combine", "filter", "transform", "output"] NodeCategory = Literal["load", "combine", "filter", "transform", "output"]
SqlSupport = Literal["full", "partial", "none"] SqlSupport = Literal["full", "partial", "none"]
NodePortDefinition = DefinitionPort
NodeConfigField = DefinitionConfigField
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class NodePortDefinition: class NodeTypeDefinition(DefinitionNodeType):
id: str
label: str
required: bool = True
multiple: bool = False
minimum_connections: int = 1
@dataclass(frozen=True, slots=True)
class NodeConfigField:
id: str
label: str
kind: str
required: bool = False
description: str | None = None
options: tuple[tuple[str, str], ...] = ()
@dataclass(frozen=True, slots=True)
class NodeTypeDefinition:
type: str
category: NodeCategory
label: str
description: str
icon: str
input_ports: tuple[NodePortDefinition, ...] = ()
output_ports: tuple[NodePortDefinition, ...] = (
NodePortDefinition(id="output", label="Output"),
)
config_fields: tuple[NodeConfigField, ...] = ()
default_config: dict[str, Any] = field(default_factory=dict)
sql_support: SqlSupport = "full" sql_support: SqlSupport = "full"
NODE_LIBRARY = ( _NODE_TYPES = (
NodeTypeDefinition( NodeTypeDefinition(
type="source.inline", type="source.inline",
category="load", category="load",
@@ -59,18 +40,29 @@ NODE_LIBRARY = (
NodeTypeDefinition( NodeTypeDefinition(
type="source.reference", type="source.reference",
category="load", category="load",
label="Connector source", label="Datasource",
description="Load a bounded, fingerprinted table exposed by Connectors.", description="Load a bounded, fingerprinted state from the Datasources catalogue.",
icon="database", icon="database",
config_fields=( config_fields=(
NodeConfigField(id="source_ref", label="Source", kind="tabular_source", required=True), NodeConfigField(id="source_ref", label="Datasource", kind="datasource", required=True),
NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True), NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True),
NodeConfigField(id="expected_fingerprint", label="Expected fingerprint", kind="readonly"), NodeConfigField(id="expected_fingerprint", label="Expected fingerprint", kind="readonly"),
NodeConfigField(
id="consistency",
label="State",
kind="select",
options=(
("current", "Current"),
("live", "Live"),
("frozen", "Latest frozen"),
),
),
), ),
default_config={ default_config={
"source_ref": "", "source_ref": "",
"source_name": "connector_source", "source_name": "datasource",
"expected_fingerprint": "", "expected_fingerprint": "",
"consistency": "current",
}, },
), ),
NodeTypeDefinition( NodeTypeDefinition(
@@ -278,7 +270,6 @@ NODE_LIBRARY = (
), ),
) )
NODE_TYPES = {definition.type: definition for definition in NODE_LIBRARY}
CATEGORY_LABELS = { CATEGORY_LABELS = {
"load": "Load", "load": "Load",
"combine": "Combine", "combine": "Combine",
@@ -286,6 +277,36 @@ CATEGORY_LABELS = {
"transform": "Transform", "transform": "Transform",
"output": "Output", "output": "Output",
} }
DATAFLOW_GRAPH_LIBRARY = DefinitionGraphLibrary(
id="dataflow",
version="0.1.0",
category_labels=CATEGORY_LABELS,
node_types=_NODE_TYPES,
constraints=DefinitionGraphConstraints(
max_nodes=100,
max_edges=200,
allow_cycles=False,
require_connected=True,
node_counts=(
DefinitionNodeCountConstraint(
code="graph.source_count",
label="source",
minimum=1,
maximum=10,
type_prefixes=("source.",),
),
DefinitionNodeCountConstraint(
code="graph.output_count",
label="output",
minimum=1,
maximum=1,
node_types=("output",),
),
),
),
)
NODE_LIBRARY: tuple[NodeTypeDefinition, ...] = _NODE_TYPES
NODE_TYPES = {definition.type: definition for definition in NODE_LIBRARY}
def node_definition(node_type: str) -> NodeTypeDefinition | None: def node_definition(node_type: str) -> NodeTypeDefinition | None:
@@ -294,6 +315,7 @@ def node_definition(node_type: str) -> NodeTypeDefinition | None:
__all__ = [ __all__ = [
"CATEGORY_LABELS", "CATEGORY_LABELS",
"DATAFLOW_GRAPH_LIBRARY",
"NODE_LIBRARY", "NODE_LIBRARY",
"NODE_TYPES", "NODE_TYPES",
"NodeCategory", "NodeCategory",

71
src/govoplan_dataflow/backend/router.py
View File

@@ -5,16 +5,19 @@ from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event from govoplan_core.audit.logging import audit_event
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
from govoplan_core.core.datasources import (
DatasourceAccessError,
DatasourceDescriptor,
DatasourceError,
DatasourceNotFoundError,
DatasourceStageInput,
DatasourceUnavailableError,
DatasourceValidationError,
datasource_catalogue,
datasource_lifecycle,
)
from govoplan_core.core.tabular_sources import ( from govoplan_core.core.tabular_sources import (
TabularSnapshotInput,
TabularSource,
TabularSourceAccessError,
TabularSourceError,
TabularSourceNotFoundError,
TabularSourceValidationError,
parse_tabular_csv, parse_tabular_csv,
tabular_snapshot_writer,
tabular_source_provider,
) )
from govoplan_core.db.session import get_session from govoplan_core.db.session import get_session
from govoplan_dataflow.backend.manifest import ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, WRITE_SCOPE from govoplan_dataflow.backend.manifest import ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, WRITE_SCOPE
@@ -93,12 +96,14 @@ def _http_error(exc: DataflowError) -> HTTPException:
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
def _source_http_error(exc: TabularSourceError) -> HTTPException: def _source_http_error(exc: DatasourceError) -> HTTPException:
if isinstance(exc, TabularSourceAccessError): if isinstance(exc, DatasourceAccessError):
return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
if isinstance(exc, TabularSourceNotFoundError): if isinstance(exc, DatasourceNotFoundError):
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
if isinstance(exc, TabularSourceValidationError): if isinstance(exc, DatasourceUnavailableError):
return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
if isinstance(exc, DatasourceValidationError):
return HTTPException( return HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc), detail=str(exc),
@@ -155,10 +160,10 @@ def _node_library_response() -> NodeLibraryResponse:
) )
def _source_response(source: TabularSource) -> TabularSourceResponse: def _source_response(source: DatasourceDescriptor) -> TabularSourceResponse:
return TabularSourceResponse( return TabularSourceResponse(
ref=source.ref, ref=source.ref,
provider=source.provider, provider=source.provider or "datasources",
source_name=source.source_name, source_name=source.source_name,
name=source.name, name=source.name,
description=source.description, description=source.description,
@@ -195,18 +200,18 @@ def api_list_sources(
) -> TabularSourceListResponse: ) -> TabularSourceListResponse:
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE) _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
registry = get_registry() registry = get_registry()
provider = tabular_source_provider(registry) provider = datasource_catalogue(registry)
writer = tabular_snapshot_writer(registry) writer = datasource_lifecycle(registry)
if provider is None: if provider is None:
return TabularSourceListResponse(available=False, writable=False, sources=[]) return TabularSourceListResponse(available=False, writable=False, sources=[])
try: try:
sources = provider.list_sources( sources = provider.list_datasources(
session, session,
principal, principal,
query=query, query=query,
limit=100, limit=100,
) )
except TabularSourceError as exc: except DatasourceError as exc:
raise _source_http_error(exc) from exc raise _source_http_error(exc) from exc
return TabularSourceListResponse( return TabularSourceListResponse(
available=True, available=True,
@@ -226,11 +231,11 @@ def api_create_source_snapshot(
principal: ApiPrincipal = Depends(get_api_principal), principal: ApiPrincipal = Depends(get_api_principal),
) -> TabularSourceResponse: ) -> TabularSourceResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
writer = tabular_snapshot_writer(get_registry()) writer = datasource_lifecycle(get_registry())
if writer is None: if writer is None:
raise HTTPException( raise HTTPException(
status_code=status.HTTP_409_CONFLICT, status_code=status.HTTP_409_CONFLICT,
detail="No tabular snapshot writer is available.", detail="No datasource staging provider is available.",
) )
try: try:
rows = ( rows = (
@@ -242,35 +247,49 @@ def api_create_source_snapshot(
if payload.format == "csv" if payload.format == "csv"
else tuple(payload.rows or ()) else tuple(payload.rows or ())
) )
source = writer.create_snapshot( stage = writer.create_stage(
session, session,
principal, principal,
snapshot=TabularSnapshotInput( stage=DatasourceStageInput(
name=payload.name, name=payload.name,
source_name=payload.source_name, source_name=payload.source_name,
description=payload.description, description=payload.description,
kind="upload",
mode="static",
shape="tabular",
rows=rows, rows=rows,
metadata={ provider="dataflow.upload",
provenance={
"created_via": "dataflow", "created_via": "dataflow",
"source_format": payload.format, "source_format": payload.format,
}, },
metadata={
"dataflow_convenience_import": True,
},
), ),
) )
except TabularSourceError as exc: source, materialization = writer.promote_stage(
session,
principal,
stage_ref=stage.ref,
)
except DatasourceError as exc:
raise _source_http_error(exc) from exc raise _source_http_error(exc) from exc
audit_event( audit_event(
session, session,
tenant_id=principal.tenant_id, tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None), user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id, api_key_id=principal.api_key_id,
action="dataflow.source_snapshot.created", action="dataflow.datasource.created",
object_type="tabular_source", object_type="datasource",
object_id=source.ref, object_id=source.ref,
details={ details={
"provider": source.provider, "provider": source.provider,
"source_name": source.source_name, "source_name": source.source_name,
"row_count": source.row_count, "row_count": source.row_count,
"fingerprint": source.fingerprint, "fingerprint": source.fingerprint,
"stage_ref": stage.ref,
"materialization_ref": materialization.ref,
}, },
) )
response = _source_response(source) response = _source_response(source)

31
src/govoplan_dataflow/backend/service.py
View File

@@ -6,10 +6,10 @@ from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.tabular_sources import ( from govoplan_core.core.datasources import (
TabularReadRequest, DatasourceError,
TabularSourceError, DatasourceReadRequest,
tabular_source_provider, datasource_catalogue,
) )
from govoplan_core.db.base import utcnow from govoplan_core.db.base import utcnow
from govoplan_dataflow.backend.db.models import ( from govoplan_dataflow.backend.db.models import (
@@ -381,38 +381,41 @@ def preview_pipeline(
started_at = utcnow() started_at = utcnow()
run: DataflowRun | None = None run: DataflowRun | None = None
try: try:
provider = tabular_source_provider(registry) provider = datasource_catalogue(registry)
def resolve_source(node: GraphNode, limit: int) -> ResolvedSource: def resolve_source(node: GraphNode, limit: int) -> ResolvedSource:
if provider is None: if provider is None:
raise PipelineExecutionError( raise PipelineExecutionError(
"Connector-backed preview requires the Connectors tabular-source capability.", "Datasource-backed preview requires the Datasources catalogue capability.",
node_id=node.id, node_id=node.id,
) )
if principal is None: if principal is None:
raise PipelineExecutionError( raise PipelineExecutionError(
"Connector-backed preview requires a tenant API principal.", "Datasource-backed preview requires a tenant API principal.",
node_id=node.id, node_id=node.id,
) )
try: try:
resolved = provider.read_source( resolved = provider.read_datasource(
session, session,
principal, principal,
request=TabularReadRequest( request=DatasourceReadRequest(
source_ref=str(node.config["source_ref"]), datasource_ref=str(node.config["source_ref"]),
consistency=str(
node.config.get("consistency") or "current"
), # type: ignore[arg-type]
limit=limit, limit=limit,
expected_fingerprint=_clean_optional( expected_fingerprint=_clean_optional(
node.config.get("expected_fingerprint") node.config.get("expected_fingerprint")
), ),
), ),
) )
except TabularSourceError as exc: except DatasourceError as exc:
raise PipelineExecutionError(str(exc), node_id=node.id) from exc raise PipelineExecutionError(str(exc), node_id=node.id) from exc
return ResolvedSource( return ResolvedSource(
rows=tuple(dict(row) for row in resolved.rows), rows=tuple(dict(row) for row in resolved.rows),
source_ref=resolved.source.ref, source_ref=resolved.datasource.ref,
provider=resolved.source.provider, provider=resolved.datasource.provider or "datasources",
fingerprint=resolved.source.fingerprint, fingerprint=resolved.datasource.fingerprint,
total_rows=resolved.total_rows, total_rows=resolved.total_rows,
truncated=resolved.truncated, truncated=resolved.truncated,
) )

2
tests/test_manifest.py
View File

@@ -13,7 +13,7 @@ class DataflowManifestTests(unittest.TestCase):
self.assertIsInstance(manifest, ModuleManifest) self.assertIsInstance(manifest, ModuleManifest)
self.assertEqual(manifest.id, "dataflow") self.assertEqual(manifest.id, "dataflow")
self.assertEqual(manifest.dependencies, ()) self.assertEqual(manifest.dependencies, ())
self.assertIn("connectors", manifest.optional_dependencies) self.assertIn("datasources", manifest.optional_dependencies)
self.assertIn("reporting", manifest.optional_dependencies) self.assertIn("reporting", manifest.optional_dependencies)
self.assertIn("workflow", manifest.optional_dependencies) self.assertIn("workflow", manifest.optional_dependencies)
self.assertEqual(manifest.frontend.package_name, "@govoplan/dataflow-webui") self.assertEqual(manifest.frontend.package_name, "@govoplan/dataflow-webui")

9
webui/src/api/dataflow.ts
View File

@@ -1,4 +1,5 @@
import { apiFetch, type ApiSettings } from "@govoplan/core-webui"; import { apiFetch, type ApiSettings } from "@govoplan/core-webui";
import type { DefinitionGraphNodeType } from "@govoplan/core-webui/definition-graph";
export type PipelineStatus = "draft" | "active" | "archived"; export type PipelineStatus = "draft" | "active" | "archived";
export type EditorMode = "graph" | "sql"; export type EditorMode = "graph" | "sql";
@@ -125,17 +126,11 @@ export type NodeConfigFieldDefinition = {
options: Array<[string, string]>; options: Array<[string, string]>;
}; };
export type NodeTypeDefinition = { export type NodeTypeDefinition = DefinitionGraphNodeType & {
type: string;
category: NodeCategory; category: NodeCategory;
category_label: string;
label: string;
description: string;
icon: string;
input_ports: NodePortDefinition[]; input_ports: NodePortDefinition[];
output_ports: NodePortDefinition[]; output_ports: NodePortDefinition[];
config_fields: NodeConfigFieldDefinition[]; config_fields: NodeConfigFieldDefinition[];
default_config: Record<string, unknown>;
sql_support: "full" | "partial" | "none"; sql_support: "full" | "partial" | "none";
}; };

55
webui/src/features/dataflow/DataflowCanvas.tsx
View File

@@ -13,6 +13,7 @@ import {
type Edge, type Edge,
type ReactFlowInstance type ReactFlowInstance
} from "@xyflow/react"; } from "@xyflow/react";
import { definitionConnectionError } from "@govoplan/core-webui/definition-graph";
import type { import type {
DataflowDiagnostic, DataflowDiagnostic,
NodeTypeDefinition, NodeTypeDefinition,
@@ -124,34 +125,17 @@ export default function DataflowCanvas({
}; };
const isValidConnection = (connection: Connection | Edge): boolean => { const isValidConnection = (connection: Connection | Edge): boolean => {
if (readOnly || !connection.source || !connection.target || connection.source === connection.target) return false; if (readOnly || !connection.source || !connection.target) return false;
const source = graph.nodes.find((node) => node.id === connection.source); return definitionConnectionError(
const target = graph.nodes.find((node) => node.id === connection.target); graph,
if (!source || !target || source.type === "output" || target.type.startsWith("source.")) return false; nodeLibrary,
const sourcePort = connection.sourceHandle ?? "output"; {
const targetPort = connection.targetHandle ?? "input"; source: connection.source,
const sourceDefinition = definitions.get(source.type); target: connection.target,
const targetDefinition = definitions.get(target.type); sourcePort: connection.sourceHandle,
if ( targetPort: connection.targetHandle
!sourceDefinition?.output_ports.some((port) => port.id === sourcePort)
|| !targetDefinition?.input_ports.some((port) => port.id === targetPort)
) {
return false;
} }
const duplicate = graph.edges.some( ) === null;
(edge) =>
edge.source === connection.source
&& edge.target === connection.target
&& (edge.source_port ?? "output") === sourcePort
&& (edge.target_port ?? "input") === targetPort
);
if (duplicate) return false;
const port = targetDefinition.input_ports.find((item) => item.id === targetPort);
const targetPortConnections = graph.edges.filter(
(edge) => edge.target === connection.target && (edge.target_port ?? "input") === targetPort
);
if (!port?.multiple && targetPortConnections.length) return false;
return !wouldCreateCycle(graph, connection.source, connection.target);
}; };
const onConnect = (connection: Connection) => { const onConnect = (connection: Connection) => {
@@ -234,23 +218,6 @@ export default function DataflowCanvas({
); );
} }
function wouldCreateCycle(graph: PipelineGraph, source: string, target: string): boolean {
const outgoing = new Map<string, string[]>();
graph.edges.forEach((edge) => {
outgoing.set(edge.source, [...(outgoing.get(edge.source) ?? []), edge.target]);
});
const pending = [target];
const visited = new Set<string>();
while (pending.length) {
const nodeId = pending.pop();
if (!nodeId || visited.has(nodeId)) continue;
if (nodeId === source) return true;
visited.add(nodeId);
pending.push(...(outgoing.get(nodeId) ?? []));
}
return false;
}
export function updateGraphNode( export function updateGraphNode(
graph: PipelineGraph, graph: PipelineGraph,
updatedNode: PipelineGraphNode updatedNode: PipelineGraphNode

8
webui/src/features/dataflow/DataflowPage.tsx
View File

@@ -106,8 +106,8 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
const canImportSources = canWrite const canImportSources = canWrite
&& sourceCatalogueWritable && sourceCatalogueWritable
&& ( && (
hasScope(auth, "connectors:source:write") hasScope(auth, "datasources:stage:write")
|| hasScope(auth, "connectors:source:admin") || hasScope(auth, "datasources:source:admin")
); );
const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft); const dirty = Boolean(draft) && draftFingerprint(draft) !== draftFingerprint(savedDraft);
const selectedNode = useMemo( const selectedNode = useMemo(
@@ -571,7 +571,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
<strong>Nodes</strong> <strong>Nodes</strong>
{canImportSources ? ( {canImportSources ? (
<IconButton <IconButton
label="Import tabular snapshot" label="Stage datasource"
icon={<Upload size={15} />} icon={<Upload size={15} />}
variant="ghost" variant="ghost"
onClick={() => setSnapshotOpen(true)} onClick={() => setSnapshotOpen(true)}
@@ -815,7 +815,7 @@ function SourceSnapshotDialog({
return ( return (
<Dialog <Dialog
open={open} open={open}
title="Import tabular snapshot" title="Stage datasource"
className="dataflow-source-dialog" className="dataflow-source-dialog"
onClose={() => { onClose={() => {
if (!busy) onClose(); if (!busy) onClose();

13
webui/src/features/dataflow/NodeInspector.tsx
View File

@@ -151,7 +151,7 @@ export default function NodeInspector({
disabled={readOnly || !sourceCatalogueAvailable} disabled={readOnly || !sourceCatalogueAvailable}
> >
<option value=""> <option value="">
{sourceCatalogueAvailable ? "Choose a source" : "Connectors unavailable"} {sourceCatalogueAvailable ? "Choose a datasource" : "Datasources unavailable"}
</option> </option>
{sources.map((source) => ( {sources.map((source) => (
<option key={source.ref} value={source.ref}> <option key={source.ref} value={source.ref}>
@@ -160,6 +160,17 @@ export default function NodeInspector({
))} ))}
</select> </select>
</FormField> </FormField>
<FormField label="State">
<select
value={textValue(node.config.consistency) || "current"}
onChange={(event) => updateConfig({ consistency: event.target.value })}
disabled={readOnly}
>
<option value="current">Current</option>
<option value="live">Live</option>
<option value="frozen">Latest frozen</option>
</select>
</FormField>
{textValue(node.config.expected_fingerprint) ? ( {textValue(node.config.expected_fingerprint) ? (
<FormField label="Pinned fingerprint"> <FormField label="Pinned fingerprint">
<input <input

23
webui/src/features/dataflow/model.ts
View File

@@ -7,6 +7,7 @@ import type {
PipelineStatus, PipelineStatus,
NodeTypeDefinition NodeTypeDefinition
} from "../../api/dataflow"; } from "../../api/dataflow";
import { createDefinitionGraphNode } from "@govoplan/core-webui/definition-graph";
export type PipelineDraft = { export type PipelineDraft = {
id: string | null; id: string | null;
@@ -31,12 +32,17 @@ export const FALLBACK_NODE_LIBRARY: NodeTypeDefinition[] = [
"source.reference", "source.reference",
"load", "load",
"Load", "Load",
"Connector source", "Datasource",
"Load a governed tabular source.", "Load a governed datasource state.",
"database", "database",
[], [],
output, output,
{ source_ref: "", source_name: "connector_source", expected_fingerprint: "" } {
source_ref: "",
source_name: "datasource",
expected_fingerprint: "",
consistency: "current"
}
), ),
nodeType( nodeType(
"combine.union", "combine.union",
@@ -257,16 +263,7 @@ export function newNode(
position: { x: number; y: number }, position: { x: number; y: number },
library: NodeTypeDefinition[] = FALLBACK_NODE_LIBRARY library: NodeTypeDefinition[] = FALLBACK_NODE_LIBRARY
): PipelineGraphNode { ): PipelineGraphNode {
const id = `${type.replace(".", "-")}-${crypto.randomUUID()}`; return createDefinitionGraphNode<PipelineGraphNode>(type, position, library);
const definition = library.find((item) => item.type === type);
const config = structuredClone(definition?.default_config ?? {});
return {
id,
type,
label: definition?.label ?? type,
position,
config
};
} }
function nodeType( function nodeType(

2
webui/src/module.ts
View File

@@ -14,7 +14,7 @@ export const dataflowModule: PlatformWebModule = {
optionalDependencies: [ optionalDependencies: [
"access", "access",
"audit", "audit",
"connectors", "datasources",
"files", "files",
"notifications", "notifications",
"policy", "policy",

1
webui/tsconfig.json
View File

@@ -21,6 +21,7 @@
"baseUrl": ".", "baseUrl": ".",
"paths": { "paths": {
"@govoplan/core-webui": ["../../govoplan-core/webui/src/index.ts"], "@govoplan/core-webui": ["../../govoplan-core/webui/src/index.ts"],
"@govoplan/core-webui/definition-graph": ["../../govoplan-core/webui/src/definitionGraph.ts"],
"@govoplan/core-webui/*": ["../../govoplan-core/webui/src/*"], "@govoplan/core-webui/*": ["../../govoplan-core/webui/src/*"],
"@xyflow/react": ["../../govoplan-core/webui/node_modules/@xyflow/react/dist/esm/index.d.ts"], "@xyflow/react": ["../../govoplan-core/webui/node_modules/@xyflow/react/dist/esm/index.d.ts"],
"lucide-react": ["../../govoplan-core/webui/node_modules/lucide-react/dist/lucide-react.d.ts"], "lucide-react": ["../../govoplan-core/webui/node_modules/lucide-react/dist/lucide-react.d.ts"],