feat: consume governed datasources and shared graphs
This commit is contained in:
@@ -7,7 +7,16 @@ from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from govoplan_dataflow.backend.node_library import NODE_TYPES, node_definition
|
||||
from govoplan_core.core.definition_graphs import (
|
||||
DefinitionEdge,
|
||||
DefinitionNode,
|
||||
validate_definition_graph,
|
||||
)
|
||||
from govoplan_dataflow.backend.node_library import (
|
||||
DATAFLOW_GRAPH_LIBRARY,
|
||||
NODE_TYPES,
|
||||
node_definition,
|
||||
)
|
||||
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphEdge, GraphNode, PipelineGraph
|
||||
|
||||
|
||||
@@ -47,75 +56,49 @@ def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str:
|
||||
|
||||
|
||||
def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
diagnostics: list[DataflowDiagnostic] = []
|
||||
diagnostics = [
|
||||
DataflowDiagnostic(
|
||||
severity=item.severity,
|
||||
code=item.code,
|
||||
message=item.message,
|
||||
node_id=item.node_id,
|
||||
field=item.field,
|
||||
)
|
||||
for item in validate_definition_graph(
|
||||
DATAFLOW_GRAPH_LIBRARY,
|
||||
nodes=tuple(DefinitionNode(id=node.id, type=node.type) for node in graph.nodes),
|
||||
edges=tuple(
|
||||
DefinitionEdge(
|
||||
id=edge.id,
|
||||
source=edge.source,
|
||||
target=edge.target,
|
||||
source_port=edge.source_port,
|
||||
target_port=edge.target_port,
|
||||
)
|
||||
for edge in graph.edges
|
||||
),
|
||||
)
|
||||
]
|
||||
nodes = {node.id: node for node in graph.nodes}
|
||||
if len(nodes) != len(graph.nodes):
|
||||
diagnostics.append(_error("graph.duplicate_node", "Node identifiers must be unique."))
|
||||
|
||||
edge_ids = {edge.id for edge in graph.edges}
|
||||
if len(edge_ids) != len(graph.edges):
|
||||
diagnostics.append(_error("graph.duplicate_edge", "Edge identifiers must be unique."))
|
||||
|
||||
incoming: dict[str, list[GraphEdge]] = {node_id: [] for node_id in nodes}
|
||||
outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
for edge in graph.edges:
|
||||
if edge.source not in nodes:
|
||||
diagnostics.append(
|
||||
_error("edge.unknown_source", f"Edge {edge.id!r} references an unknown source node.")
|
||||
)
|
||||
continue
|
||||
if edge.target not in nodes:
|
||||
diagnostics.append(
|
||||
_error("edge.unknown_target", f"Edge {edge.id!r} references an unknown target node.")
|
||||
)
|
||||
continue
|
||||
if edge.source == edge.target:
|
||||
diagnostics.append(
|
||||
_error("edge.self_reference", "A node cannot connect to itself.", node_id=edge.source)
|
||||
)
|
||||
if edge.source not in nodes or edge.target not in nodes or edge.source == edge.target:
|
||||
continue
|
||||
source_definition = node_definition(nodes[edge.source].type)
|
||||
target_definition = node_definition(nodes[edge.target].type)
|
||||
if source_definition and edge.source_port not in {
|
||||
port.id for port in source_definition.output_ports
|
||||
}:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"edge.unknown_source_port",
|
||||
f"Node {edge.source!r} has no output port {edge.source_port!r}.",
|
||||
node_id=edge.source,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if target_definition and edge.target_port not in {
|
||||
port.id for port in target_definition.input_ports
|
||||
}:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"edge.unknown_target_port",
|
||||
f"Node {edge.target!r} has no input port {edge.target_port!r}.",
|
||||
node_id=edge.target,
|
||||
)
|
||||
)
|
||||
if (
|
||||
source_definition is None
|
||||
or target_definition is None
|
||||
or edge.source_port not in {port.id for port in source_definition.output_ports}
|
||||
or edge.target_port not in {port.id for port in target_definition.input_ports}
|
||||
):
|
||||
continue
|
||||
outgoing[edge.source].append(edge.target)
|
||||
incoming[edge.target].append(edge)
|
||||
|
||||
if not graph.nodes:
|
||||
diagnostics.append(_error("graph.empty", "Add a source and an output before saving the pipeline."))
|
||||
return diagnostics
|
||||
|
||||
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
|
||||
output_nodes = [node for node in graph.nodes if node.type == "output"]
|
||||
if not source_nodes:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"graph.source_count",
|
||||
"A pipeline needs at least one source node.",
|
||||
)
|
||||
)
|
||||
elif len(source_nodes) > 10:
|
||||
diagnostics.append(_error("graph.source_limit", "Pipelines are limited to ten sources."))
|
||||
source_names = [
|
||||
str(node.config.get("source_name", "")).strip()
|
||||
for node in source_nodes
|
||||
@@ -137,54 +120,13 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
f"{', '.join(duplicate_source_names)}.",
|
||||
)
|
||||
)
|
||||
if len(output_nodes) != 1:
|
||||
diagnostics.append(
|
||||
_error("graph.output_count", "A pipeline must contain exactly one output node.")
|
||||
)
|
||||
|
||||
for node in graph.nodes:
|
||||
if node.type not in SUPPORTED_NODE_TYPES:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.unsupported_type",
|
||||
f"Node type {node.type!r} is not supported by this executor.",
|
||||
node_id=node.id,
|
||||
field="type",
|
||||
)
|
||||
)
|
||||
continue
|
||||
definition = node_definition(node.type)
|
||||
node_edges = incoming.get(node.id, [])
|
||||
if definition is not None:
|
||||
for port in definition.input_ports:
|
||||
connections = [
|
||||
edge
|
||||
for edge in node_edges
|
||||
if edge.target_port == port.id
|
||||
]
|
||||
minimum = port.minimum_connections if port.required else 0
|
||||
if len(connections) < minimum:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.input_required",
|
||||
f"{definition.label} requires {port.label.lower()} input.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
if not port.multiple and len(connections) > 1:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.input_multiple",
|
||||
f"{port.label} accepts only one connection.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
diagnostics.extend(_validate_node_config(node))
|
||||
|
||||
ordered, cyclic = topological_order(graph)
|
||||
if cyclic:
|
||||
diagnostics.append(_error("graph.cycle", "Pipeline edges must form an acyclic graph."))
|
||||
elif source_nodes and output_nodes:
|
||||
if not cyclic and source_nodes and len(output_nodes) == 1:
|
||||
reachable: set[str] = set()
|
||||
for source in source_nodes:
|
||||
reachable.update(_reachable_from(source.id, outgoing))
|
||||
@@ -206,7 +148,21 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
_error("graph.output_not_terminal", "The output node must be the terminal transform.")
|
||||
)
|
||||
diagnostics.extend(_validate_graph_schemas(graph, ordered=ordered))
|
||||
return diagnostics
|
||||
return _dedupe_diagnostics(diagnostics)
|
||||
|
||||
|
||||
def _dedupe_diagnostics(
|
||||
diagnostics: list[DataflowDiagnostic],
|
||||
) -> list[DataflowDiagnostic]:
|
||||
seen: set[tuple[str, str | None, str | None]] = set()
|
||||
result: list[DataflowDiagnostic] = []
|
||||
for item in diagnostics:
|
||||
key = (item.code, item.node_id, item.field)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]:
|
||||
@@ -577,7 +533,7 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"source.reference_required",
|
||||
"Choose a connector source.",
|
||||
"Choose a datasource.",
|
||||
node_id=node.id,
|
||||
field="source_ref",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user