from __future__ import annotations import hashlib import json from collections import deque from typing import Any from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphNode, PipelineGraph SUPPORTED_NODE_TYPES = frozenset( { "source.inline", "source.reference", "filter", "select", "aggregate", "sort", "limit", "output", } ) FILTER_OPERATORS = frozenset( {"eq", "ne", "gt", "gte", "lt", "lte", "contains", "is_null", "not_null"} ) AGGREGATE_FUNCTIONS = frozenset({"count", "sum", "avg", "min", "max"}) def canonical_graph_payload(graph: PipelineGraph) -> dict[str, Any]: return graph.model_dump(mode="json", exclude_none=True) def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str: payload = { "graph": canonical_graph_payload(graph), "sql_text": (sql_text or "").strip() or None, } encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]: diagnostics: list[DataflowDiagnostic] = [] nodes = {node.id: node for node in graph.nodes} if len(nodes) != len(graph.nodes): diagnostics.append(_error("graph.duplicate_node", "Node identifiers must be unique.")) edge_ids = {edge.id for edge in graph.edges} if len(edge_ids) != len(graph.edges): diagnostics.append(_error("graph.duplicate_edge", "Edge identifiers must be unique.")) incoming: dict[str, list[str]] = {node_id: [] for node_id in nodes} outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes} for edge in graph.edges: if edge.source not in nodes: diagnostics.append( _error("edge.unknown_source", f"Edge {edge.id!r} references an unknown source node.") ) continue if edge.target not in nodes: diagnostics.append( _error("edge.unknown_target", f"Edge {edge.id!r} references an unknown target node.") ) continue if edge.source == edge.target: diagnostics.append( _error("edge.self_reference", "A node cannot connect to itself.", node_id=edge.source) ) continue outgoing[edge.source].append(edge.target) incoming[edge.target].append(edge.source) if not graph.nodes: diagnostics.append(_error("graph.empty", "Add a source and an output before saving the pipeline.")) return diagnostics source_nodes = [node for node in graph.nodes if node.type.startswith("source.")] output_nodes = [node for node in graph.nodes if node.type == "output"] if len(source_nodes) != 1: diagnostics.append( _error( "graph.source_count", "The first release supports exactly one source node per pipeline.", ) ) if len(output_nodes) != 1: diagnostics.append( _error("graph.output_count", "A pipeline must contain exactly one output node.") ) for node in graph.nodes: if node.type not in SUPPORTED_NODE_TYPES: diagnostics.append( _error( "node.unsupported_type", f"Node type {node.type!r} is not supported by this executor.", node_id=node.id, field="type", ) ) continue if node.type.startswith("source."): if incoming.get(node.id): diagnostics.append( _error("node.source_has_input", "Source nodes cannot have incoming edges.", node_id=node.id) ) elif len(incoming.get(node.id, [])) != 1: diagnostics.append( _error( "node.input_count", "This transform requires exactly one incoming edge.", node_id=node.id, ) ) diagnostics.extend(_validate_node_config(node)) ordered, cyclic = topological_order(graph) if cyclic: diagnostics.append(_error("graph.cycle", "Pipeline edges must form an acyclic graph.")) elif source_nodes and output_nodes: reachable = _reachable_from(source_nodes[0].id, outgoing) if len(reachable) != len(nodes): diagnostics.append( _error("graph.disconnected", "Every node must be connected to the pipeline source.") ) reaches_output = _reachable_from(output_nodes[0].id, incoming) if len(reaches_output) != len(nodes): diagnostics.append( _error("graph.dead_end", "Every node must lead to the pipeline output.") ) if ordered and ordered[-1] != output_nodes[0].id: diagnostics.append( _error("graph.output_not_terminal", "The output node must be the terminal transform.") ) return diagnostics def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]: node_ids = [node.id for node in graph.nodes] incoming_count = {node_id: 0 for node_id in node_ids} outgoing: dict[str, list[str]] = {node_id: [] for node_id in node_ids} for edge in graph.edges: if edge.source in outgoing and edge.target in incoming_count and edge.source != edge.target: outgoing[edge.source].append(edge.target) incoming_count[edge.target] += 1 ready = deque(node_id for node_id in node_ids if incoming_count[node_id] == 0) ordered: list[str] = [] while ready: node_id = ready.popleft() ordered.append(node_id) for target in outgoing[node_id]: incoming_count[target] -= 1 if incoming_count[target] == 0: ready.append(target) return ordered, len(ordered) != len(node_ids) def graph_input_map(graph: PipelineGraph) -> dict[str, str]: return {edge.target: edge.source for edge in graph.edges} def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]: seen: set[str] = set() pending = [start] while pending: node_id = pending.pop() if node_id in seen: continue seen.add(node_id) pending.extend(adjacency.get(node_id, ())) return seen def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]: config = node.config diagnostics: list[DataflowDiagnostic] = [] if node.type in {"source.inline", "source.reference"}: source_name = config.get("source_name") if not isinstance(source_name, str) or not source_name.strip(): diagnostics.append( _error( "source.name_required", "A source needs a logical SQL name.", node_id=node.id, field="source_name", ) ) if node.type == "source.reference": return diagnostics rows = config.get("rows") if not isinstance(rows, list): diagnostics.append( _error("source.rows_required", "Inline source rows must be a list.", node_id=node.id, field="rows") ) elif len(rows) > 250: diagnostics.append( _error("source.row_limit", "Inline sources are limited to 250 rows.", node_id=node.id, field="rows") ) elif any(not isinstance(row, dict) for row in rows): diagnostics.append( _error("source.row_shape", "Every inline source row must be an object.", node_id=node.id, field="rows") ) if len(json.dumps(config, default=str).encode("utf-8")) > 512_000: diagnostics.append( _error("source.byte_limit", "Inline source configuration is limited to 512 KiB.", node_id=node.id) ) elif node.type == "filter": if not _non_empty_text(config.get("column")): diagnostics.append(_node_field_error(node, "filter.column", "Choose a column.", "column")) operator = config.get("operator") if operator not in FILTER_OPERATORS: diagnostics.append( _node_field_error(node, "filter.operator", "Choose a supported comparison operator.", "operator") ) if operator not in {"is_null", "not_null"} and "value" not in config: diagnostics.append(_node_field_error(node, "filter.value", "Enter a comparison value.", "value")) elif node.type == "select": fields = config.get("fields") if not isinstance(fields, list) or not fields: diagnostics.append( _node_field_error(node, "select.fields", "Select at least one output column.", "fields") ) else: for field in fields: column = field if isinstance(field, str) else field.get("column") if isinstance(field, dict) else None if not _non_empty_text(column): diagnostics.append( _node_field_error(node, "select.field", "Selected fields need a source column.", "fields") ) break elif node.type == "aggregate": group_by = config.get("group_by", []) aggregates = config.get("aggregates") if not isinstance(group_by, list) or any(not _non_empty_text(item) for item in group_by): diagnostics.append( _node_field_error(node, "aggregate.group_by", "Group-by columns must be named.", "group_by") ) if not isinstance(aggregates, list) or not aggregates: diagnostics.append( _node_field_error(node, "aggregate.required", "Add at least one aggregate.", "aggregates") ) else: for aggregate in aggregates: if not isinstance(aggregate, dict) or aggregate.get("function") not in AGGREGATE_FUNCTIONS: diagnostics.append( _node_field_error( node, "aggregate.function", "Choose COUNT, SUM, AVG, MIN, or MAX.", "aggregates", ) ) break if aggregate.get("function") != "count" and not _non_empty_text(aggregate.get("column")): diagnostics.append( _node_field_error( node, "aggregate.column", "This aggregate needs a source column.", "aggregates", ) ) break if not _non_empty_text(aggregate.get("alias")): diagnostics.append( _node_field_error(node, "aggregate.alias", "Every aggregate needs an alias.", "aggregates") ) break elif node.type == "sort": fields = config.get("fields") if not isinstance(fields, list) or not fields: diagnostics.append(_node_field_error(node, "sort.fields", "Add at least one sort field.", "fields")) elif any( not isinstance(item, dict) or not _non_empty_text(item.get("column")) or item.get("direction", "asc") not in {"asc", "desc"} for item in fields ): diagnostics.append( _node_field_error(node, "sort.field", "Sort fields need a column and direction.", "fields") ) elif node.type == "limit": count = config.get("count") if not isinstance(count, int) or isinstance(count, bool) or not 1 <= count <= 100_000: diagnostics.append( _node_field_error(node, "limit.count", "Limit must be between 1 and 100,000.", "count") ) return diagnostics def _non_empty_text(value: object) -> bool: return isinstance(value, str) and bool(value.strip()) def _node_field_error(node: GraphNode, code: str, message: str, field: str) -> DataflowDiagnostic: return _error(code, message, node_id=node.id, field=field) def _error( code: str, message: str, *, node_id: str | None = None, field: str | None = None, ) -> DataflowDiagnostic: return DataflowDiagnostic( severity="error", code=code, message=message, node_id=node_id, field=field, ) __all__ = [ "AGGREGATE_FUNCTIONS", "FILTER_OPERATORS", "SUPPORTED_NODE_TYPES", "canonical_graph_payload", "definition_hash", "graph_input_map", "topological_order", "validate_graph", ]