Initialize governed Dataflow module
This commit is contained in:
325
src/govoplan_dataflow/backend/graph.py
Normal file
325
src/govoplan_dataflow/backend/graph.py
Normal file
@@ -0,0 +1,325 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
|
||||
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphNode, PipelineGraph
|
||||
|
||||
|
||||
SUPPORTED_NODE_TYPES = frozenset(
|
||||
{
|
||||
"source.inline",
|
||||
"source.reference",
|
||||
"filter",
|
||||
"select",
|
||||
"aggregate",
|
||||
"sort",
|
||||
"limit",
|
||||
"output",
|
||||
}
|
||||
)
|
||||
FILTER_OPERATORS = frozenset(
|
||||
{"eq", "ne", "gt", "gte", "lt", "lte", "contains", "is_null", "not_null"}
|
||||
)
|
||||
AGGREGATE_FUNCTIONS = frozenset({"count", "sum", "avg", "min", "max"})
|
||||
|
||||
|
||||
def canonical_graph_payload(graph: PipelineGraph) -> dict[str, Any]:
|
||||
return graph.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
|
||||
def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str:
|
||||
payload = {
|
||||
"graph": canonical_graph_payload(graph),
|
||||
"sql_text": (sql_text or "").strip() or None,
|
||||
}
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
diagnostics: list[DataflowDiagnostic] = []
|
||||
nodes = {node.id: node for node in graph.nodes}
|
||||
if len(nodes) != len(graph.nodes):
|
||||
diagnostics.append(_error("graph.duplicate_node", "Node identifiers must be unique."))
|
||||
|
||||
edge_ids = {edge.id for edge in graph.edges}
|
||||
if len(edge_ids) != len(graph.edges):
|
||||
diagnostics.append(_error("graph.duplicate_edge", "Edge identifiers must be unique."))
|
||||
|
||||
incoming: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
for edge in graph.edges:
|
||||
if edge.source not in nodes:
|
||||
diagnostics.append(
|
||||
_error("edge.unknown_source", f"Edge {edge.id!r} references an unknown source node.")
|
||||
)
|
||||
continue
|
||||
if edge.target not in nodes:
|
||||
diagnostics.append(
|
||||
_error("edge.unknown_target", f"Edge {edge.id!r} references an unknown target node.")
|
||||
)
|
||||
continue
|
||||
if edge.source == edge.target:
|
||||
diagnostics.append(
|
||||
_error("edge.self_reference", "A node cannot connect to itself.", node_id=edge.source)
|
||||
)
|
||||
continue
|
||||
outgoing[edge.source].append(edge.target)
|
||||
incoming[edge.target].append(edge.source)
|
||||
|
||||
if not graph.nodes:
|
||||
diagnostics.append(_error("graph.empty", "Add a source and an output before saving the pipeline."))
|
||||
return diagnostics
|
||||
|
||||
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
|
||||
output_nodes = [node for node in graph.nodes if node.type == "output"]
|
||||
if len(source_nodes) != 1:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"graph.source_count",
|
||||
"The first release supports exactly one source node per pipeline.",
|
||||
)
|
||||
)
|
||||
if len(output_nodes) != 1:
|
||||
diagnostics.append(
|
||||
_error("graph.output_count", "A pipeline must contain exactly one output node.")
|
||||
)
|
||||
|
||||
for node in graph.nodes:
|
||||
if node.type not in SUPPORTED_NODE_TYPES:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.unsupported_type",
|
||||
f"Node type {node.type!r} is not supported by this executor.",
|
||||
node_id=node.id,
|
||||
field="type",
|
||||
)
|
||||
)
|
||||
continue
|
||||
if node.type.startswith("source."):
|
||||
if incoming.get(node.id):
|
||||
diagnostics.append(
|
||||
_error("node.source_has_input", "Source nodes cannot have incoming edges.", node_id=node.id)
|
||||
)
|
||||
elif len(incoming.get(node.id, [])) != 1:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.input_count",
|
||||
"This transform requires exactly one incoming edge.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
diagnostics.extend(_validate_node_config(node))
|
||||
|
||||
ordered, cyclic = topological_order(graph)
|
||||
if cyclic:
|
||||
diagnostics.append(_error("graph.cycle", "Pipeline edges must form an acyclic graph."))
|
||||
elif source_nodes and output_nodes:
|
||||
reachable = _reachable_from(source_nodes[0].id, outgoing)
|
||||
if len(reachable) != len(nodes):
|
||||
diagnostics.append(
|
||||
_error("graph.disconnected", "Every node must be connected to the pipeline source.")
|
||||
)
|
||||
reaches_output = _reachable_from(output_nodes[0].id, incoming)
|
||||
if len(reaches_output) != len(nodes):
|
||||
diagnostics.append(
|
||||
_error("graph.dead_end", "Every node must lead to the pipeline output.")
|
||||
)
|
||||
if ordered and ordered[-1] != output_nodes[0].id:
|
||||
diagnostics.append(
|
||||
_error("graph.output_not_terminal", "The output node must be the terminal transform.")
|
||||
)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]:
|
||||
node_ids = [node.id for node in graph.nodes]
|
||||
incoming_count = {node_id: 0 for node_id in node_ids}
|
||||
outgoing: dict[str, list[str]] = {node_id: [] for node_id in node_ids}
|
||||
for edge in graph.edges:
|
||||
if edge.source in outgoing and edge.target in incoming_count and edge.source != edge.target:
|
||||
outgoing[edge.source].append(edge.target)
|
||||
incoming_count[edge.target] += 1
|
||||
|
||||
ready = deque(node_id for node_id in node_ids if incoming_count[node_id] == 0)
|
||||
ordered: list[str] = []
|
||||
while ready:
|
||||
node_id = ready.popleft()
|
||||
ordered.append(node_id)
|
||||
for target in outgoing[node_id]:
|
||||
incoming_count[target] -= 1
|
||||
if incoming_count[target] == 0:
|
||||
ready.append(target)
|
||||
return ordered, len(ordered) != len(node_ids)
|
||||
|
||||
|
||||
def graph_input_map(graph: PipelineGraph) -> dict[str, str]:
|
||||
return {edge.target: edge.source for edge in graph.edges}
|
||||
|
||||
|
||||
def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]:
|
||||
seen: set[str] = set()
|
||||
pending = [start]
|
||||
while pending:
|
||||
node_id = pending.pop()
|
||||
if node_id in seen:
|
||||
continue
|
||||
seen.add(node_id)
|
||||
pending.extend(adjacency.get(node_id, ()))
|
||||
return seen
|
||||
|
||||
|
||||
def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
||||
config = node.config
|
||||
diagnostics: list[DataflowDiagnostic] = []
|
||||
if node.type in {"source.inline", "source.reference"}:
|
||||
source_name = config.get("source_name")
|
||||
if not isinstance(source_name, str) or not source_name.strip():
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"source.name_required",
|
||||
"A source needs a logical SQL name.",
|
||||
node_id=node.id,
|
||||
field="source_name",
|
||||
)
|
||||
)
|
||||
if node.type == "source.reference":
|
||||
return diagnostics
|
||||
rows = config.get("rows")
|
||||
if not isinstance(rows, list):
|
||||
diagnostics.append(
|
||||
_error("source.rows_required", "Inline source rows must be a list.", node_id=node.id, field="rows")
|
||||
)
|
||||
elif len(rows) > 250:
|
||||
diagnostics.append(
|
||||
_error("source.row_limit", "Inline sources are limited to 250 rows.", node_id=node.id, field="rows")
|
||||
)
|
||||
elif any(not isinstance(row, dict) for row in rows):
|
||||
diagnostics.append(
|
||||
_error("source.row_shape", "Every inline source row must be an object.", node_id=node.id, field="rows")
|
||||
)
|
||||
if len(json.dumps(config, default=str).encode("utf-8")) > 512_000:
|
||||
diagnostics.append(
|
||||
_error("source.byte_limit", "Inline source configuration is limited to 512 KiB.", node_id=node.id)
|
||||
)
|
||||
elif node.type == "filter":
|
||||
if not _non_empty_text(config.get("column")):
|
||||
diagnostics.append(_node_field_error(node, "filter.column", "Choose a column.", "column"))
|
||||
operator = config.get("operator")
|
||||
if operator not in FILTER_OPERATORS:
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "filter.operator", "Choose a supported comparison operator.", "operator")
|
||||
)
|
||||
if operator not in {"is_null", "not_null"} and "value" not in config:
|
||||
diagnostics.append(_node_field_error(node, "filter.value", "Enter a comparison value.", "value"))
|
||||
elif node.type == "select":
|
||||
fields = config.get("fields")
|
||||
if not isinstance(fields, list) or not fields:
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "select.fields", "Select at least one output column.", "fields")
|
||||
)
|
||||
else:
|
||||
for field in fields:
|
||||
column = field if isinstance(field, str) else field.get("column") if isinstance(field, dict) else None
|
||||
if not _non_empty_text(column):
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "select.field", "Selected fields need a source column.", "fields")
|
||||
)
|
||||
break
|
||||
elif node.type == "aggregate":
|
||||
group_by = config.get("group_by", [])
|
||||
aggregates = config.get("aggregates")
|
||||
if not isinstance(group_by, list) or any(not _non_empty_text(item) for item in group_by):
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "aggregate.group_by", "Group-by columns must be named.", "group_by")
|
||||
)
|
||||
if not isinstance(aggregates, list) or not aggregates:
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "aggregate.required", "Add at least one aggregate.", "aggregates")
|
||||
)
|
||||
else:
|
||||
for aggregate in aggregates:
|
||||
if not isinstance(aggregate, dict) or aggregate.get("function") not in AGGREGATE_FUNCTIONS:
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"aggregate.function",
|
||||
"Choose COUNT, SUM, AVG, MIN, or MAX.",
|
||||
"aggregates",
|
||||
)
|
||||
)
|
||||
break
|
||||
if aggregate.get("function") != "count" and not _non_empty_text(aggregate.get("column")):
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"aggregate.column",
|
||||
"This aggregate needs a source column.",
|
||||
"aggregates",
|
||||
)
|
||||
)
|
||||
break
|
||||
if not _non_empty_text(aggregate.get("alias")):
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "aggregate.alias", "Every aggregate needs an alias.", "aggregates")
|
||||
)
|
||||
break
|
||||
elif node.type == "sort":
|
||||
fields = config.get("fields")
|
||||
if not isinstance(fields, list) or not fields:
|
||||
diagnostics.append(_node_field_error(node, "sort.fields", "Add at least one sort field.", "fields"))
|
||||
elif any(
|
||||
not isinstance(item, dict)
|
||||
or not _non_empty_text(item.get("column"))
|
||||
or item.get("direction", "asc") not in {"asc", "desc"}
|
||||
for item in fields
|
||||
):
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "sort.field", "Sort fields need a column and direction.", "fields")
|
||||
)
|
||||
elif node.type == "limit":
|
||||
count = config.get("count")
|
||||
if not isinstance(count, int) or isinstance(count, bool) or not 1 <= count <= 100_000:
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "limit.count", "Limit must be between 1 and 100,000.", "count")
|
||||
)
|
||||
return diagnostics
|
||||
|
||||
|
||||
def _non_empty_text(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip())
|
||||
|
||||
|
||||
def _node_field_error(node: GraphNode, code: str, message: str, field: str) -> DataflowDiagnostic:
|
||||
return _error(code, message, node_id=node.id, field=field)
|
||||
|
||||
|
||||
def _error(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
node_id: str | None = None,
|
||||
field: str | None = None,
|
||||
) -> DataflowDiagnostic:
|
||||
return DataflowDiagnostic(
|
||||
severity="error",
|
||||
code=code,
|
||||
message=message,
|
||||
node_id=node_id,
|
||||
field=field,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AGGREGATE_FUNCTIONS",
|
||||
"FILTER_OPERATORS",
|
||||
"SUPPORTED_NODE_TYPES",
|
||||
"canonical_graph_payload",
|
||||
"definition_hash",
|
||||
"graph_input_map",
|
||||
"topological_order",
|
||||
"validate_graph",
|
||||
]
|
||||
Reference in New Issue
Block a user