from __future__ import annotations import hashlib import json import time from collections import defaultdict from dataclasses import dataclass from decimal import Decimal from typing import Any, Callable from govoplan_dataflow.backend.expressions import ( convert_value, evaluate_expression, parse_expression, ) from govoplan_dataflow.backend.graph import graph_inputs_by_port, topological_order, validate_graph from govoplan_dataflow.backend.operator_registry import ( OPERATOR_REGISTRY, OperatorExecutionContext, OperatorExecutionResult, ) from govoplan_dataflow.backend.schemas import ( DataflowDiagnostic, GraphNode, NodePreviewDiagnostic, NodePreviewResult, PipelineGraph, PreviewColumn, ) from govoplan_dataflow.backend.subflows import substitute_parameters EXECUTOR_VERSION = "dataflow-preview-v2" MAX_EXECUTION_SECONDS = 2.0 MAX_RESULT_BYTES = 1_000_000 MAX_SOURCE_ROWS = 250 MAX_INTERMEDIATE_ROWS = 10_000 class PipelineExecutionError(RuntimeError): def __init__( self, message: str, *, node_id: str | None = None, node_diagnostics: tuple[NodePreviewDiagnostic, ...] = (), source_fingerprints: tuple[dict[str, Any], ...] = (), input_row_count: int = 0, diagnostics: tuple[DataflowDiagnostic, ...] = (), node_preview: NodePreviewResult | None = None, ) -> None: super().__init__(message) self.node_id = node_id self.node_diagnostics = node_diagnostics self.source_fingerprints = source_fingerprints self.input_row_count = input_row_count self.diagnostics = diagnostics self.node_preview = node_preview @dataclass(frozen=True) class PipelineExecutionResult: rows: list[dict[str, Any]] total_rows: int truncated: bool columns: list[PreviewColumn] diagnostics: list[DataflowDiagnostic] node_diagnostics: list[NodePreviewDiagnostic] node_preview: NodePreviewResult | None source_fingerprints: list[dict[str, Any]] input_row_count: int @dataclass(frozen=True) class ResolvedSource: rows: tuple[dict[str, Any], ...] source_ref: str provider: str fingerprint: str total_rows: int truncated: bool = False SourceResolver = Callable[[GraphNode, int], ResolvedSource] def execute_preview( graph: PipelineGraph, *, row_limit: int, source_resolver: SourceResolver | None = None, preview_node_id: str | None = None, _execution_depth: int = 0, ) -> PipelineExecutionResult: if _execution_depth > 5: raise PipelineExecutionError("Subflows are limited to five nested levels.") validation = validate_graph(graph) errors = [item for item in validation if item.severity == "error"] if errors: raise PipelineExecutionError(errors[0].message, node_id=errors[0].node_id) node_by_id = {node.id: node for node in graph.nodes} if preview_node_id is not None and preview_node_id not in node_by_id: raise PipelineExecutionError( "The requested preview node is not part of this pipeline.", node_id=preview_node_id, ) inputs = graph_inputs_by_port(graph) ordered, cyclic = topological_order(graph) if cyclic: raise PipelineExecutionError("Pipeline graph contains a cycle") outputs: dict[str, list[dict[str, Any]]] = {} node_diagnostics: list[NodePreviewDiagnostic] = [] source_fingerprints: list[dict[str, Any]] = [] started = time.monotonic() input_row_count = 0 for node_id in ordered: node = node_by_id[node_id] node_started = time.monotonic() node_inputs = inputs.get(node_id, {}) input_sets = [ outputs[source_id] for port_sources in node_inputs.values() for source_id in port_sources ] input_rows = input_sets[0] if len(input_sets) == 1 else [] node_messages: list[str] = [] if time.monotonic() - started > MAX_EXECUTION_SECONDS: failed = NodePreviewDiagnostic( node_id=node.id, status="failed", input_rows=sum(len(rows) for rows in input_sets), output_rows=0, duration_ms=0, columns=[], messages=["Preview exceeded the two-second execution limit"], ) raise PipelineExecutionError( "Preview exceeded the two-second execution limit", node_id=node.id, node_diagnostics=tuple([*node_diagnostics, failed]), source_fingerprints=tuple(source_fingerprints), input_row_count=input_row_count, diagnostics=tuple(item for item in validation if item.severity != "error"), node_preview=_node_preview(outputs, preview_node_id, row_limit), ) try: executor = OPERATOR_REGISTRY.executor(node.type) if executor is None: raise PipelineExecutionError( f"Node type {node.type!r} has no registered executor.", node_id=node.id, ) execution = executor( OperatorExecutionContext( node=node, inputs_by_port=node_inputs, outputs=outputs, input_sets=input_sets, input_rows=input_rows, source_resolver=source_resolver, execution_depth=_execution_depth, ) ) output_rows = execution.rows input_row_count += execution.input_row_count source_fingerprints.extend(execution.source_fingerprints) node_messages.extend(execution.messages) if node.type == "source.reference" and execution.messages: validation.extend( DataflowDiagnostic( severity="warning", code="source.preview_truncated", message=message, node_id=node.id, ) for message in execution.messages ) if len(json.dumps(output_rows, default=str).encode("utf-8")) > MAX_RESULT_BYTES: raise PipelineExecutionError( "A preview node exceeded the one-megabyte result limit.", node_id=node.id, ) except (PipelineExecutionError, ArithmeticError, KeyError, TypeError, ValueError) as exc: execution_error = ( exc if isinstance(exc, PipelineExecutionError) else PipelineExecutionError(str(exc), node_id=node.id) ) failed = NodePreviewDiagnostic( node_id=execution_error.node_id or node.id, status="failed", input_rows=sum(len(rows) for rows in input_sets), output_rows=0, duration_ms=round((time.monotonic() - node_started) * 1000, 3), columns=[], messages=[str(execution_error)], ) raise PipelineExecutionError( str(execution_error), node_id=execution_error.node_id or node.id, node_diagnostics=tuple([*node_diagnostics, failed]), source_fingerprints=tuple(source_fingerprints), input_row_count=input_row_count, diagnostics=tuple(item for item in validation if item.severity != "error"), node_preview=_node_preview(outputs, preview_node_id, row_limit), ) from exc outputs[node.id] = output_rows node_diagnostics.append( NodePreviewDiagnostic( node_id=node.id, status="succeeded", input_rows=sum(len(rows) for rows in input_sets), output_rows=len(output_rows), duration_ms=round((time.monotonic() - node_started) * 1000, 3), columns=infer_columns(output_rows), messages=node_messages, ) ) output_node = next(node for node in graph.nodes if node.type == "output") all_rows = outputs[output_node.id] rows = all_rows[:row_limit] return PipelineExecutionResult( rows=rows, total_rows=len(all_rows), truncated=len(rows) < len(all_rows), columns=infer_columns(all_rows), diagnostics=[item for item in validation if item.severity != "error"], node_diagnostics=node_diagnostics, node_preview=_node_preview(outputs, preview_node_id, row_limit), source_fingerprints=source_fingerprints, input_row_count=input_row_count, ) def _node_preview( outputs: dict[str, list[dict[str, Any]]], node_id: str | None, row_limit: int, ) -> NodePreviewResult | None: if node_id is None or node_id not in outputs: return None all_rows = outputs[node_id] rows = [dict(row) for row in all_rows[:row_limit]] return NodePreviewResult( node_id=node_id, columns=infer_columns(all_rows), rows=rows, total_rows=len(all_rows), truncated=len(rows) < len(all_rows), ) def infer_columns(rows: list[dict[str, Any]]) -> list[PreviewColumn]: names: list[str] = [] for row in rows: for name in row: if name not in names: names.append(name) columns: list[PreviewColumn] = [] for name in names: values = [row.get(name) for row in rows] concrete = [value for value in values if value is not None] inferred = _type_name(concrete[0]) if concrete else "unknown" if any(_type_name(value) != inferred for value in concrete[1:]): inferred = "mixed" columns.append( PreviewColumn( name=name, type=inferred, nullable=len(concrete) != len(values), ) ) return columns def _union_rows( inputs: list[list[dict[str, Any]]], config: dict[str, Any], ) -> list[dict[str, Any]]: rows = [dict(row) for input_rows in inputs for row in input_rows] if config.get("mode", "all") == "distinct": return _distinct_rows(rows, {}) return rows def _join_rows( left_rows: list[dict[str, Any]], right_rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: left_keys = [str(item) for item in config["left_keys"]] right_keys = [str(item) for item in config["right_keys"]] join_type = str(config.get("join_type", "inner")) right_prefix = str(config.get("right_prefix", "right_")) left_columns = _ordered_columns(left_rows) right_columns = _ordered_columns(right_rows) right_index: dict[tuple[Any, ...], list[tuple[int, dict[str, Any]]]] = defaultdict(list) for index, row in enumerate(right_rows): key = _join_key(row, right_keys) if key is not None: right_index[key].append((index, row)) output: list[dict[str, Any]] = [] matched_right: set[int] = set() for left_row in left_rows: key = _join_key(left_row, left_keys) matches = right_index.get(key, ()) if key is not None else () if matches: for right_index_value, right_row in matches: matched_right.add(right_index_value) output.append( _merge_join_rows( left_row, right_row, left_columns=left_columns, right_columns=right_columns, right_prefix=right_prefix, ) ) _guard_intermediate_size(output, node_id=node_id) elif join_type in {"left", "full"}: output.append( _merge_join_rows( left_row, None, left_columns=left_columns, right_columns=right_columns, right_prefix=right_prefix, ) ) if join_type in {"right", "full"}: for index, right_row in enumerate(right_rows): if index in matched_right: continue output.append( _merge_join_rows( None, right_row, left_columns=left_columns, right_columns=right_columns, right_prefix=right_prefix, ) ) _guard_intermediate_size(output, node_id=node_id) return output def _distinct_rows( rows: list[dict[str, Any]], config: dict[str, Any], ) -> list[dict[str, Any]]: columns = [str(item) for item in config.get("columns", [])] seen: set[str] = set() output: list[dict[str, Any]] = [] for row in rows: value = {column: row.get(column) for column in columns} if columns else row identity = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) if identity in seen: continue seen.add(identity) output.append(dict(row)) return output def _derive_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: target = str(config["target_column"]) operation = str(config["operation"]) columns = [str(item) for item in config["source_columns"]] separator = str(config.get("separator", " ")) output: list[dict[str, Any]] = [] for row in rows: values = [row.get(column) for column in columns] try: derived = _derive_value(operation, values, separator=separator) except (ArithmeticError, TypeError, ValueError) as exc: raise PipelineExecutionError( f"Cannot derive {target!r} with {operation!r}: {exc}", node_id=node_id, ) from exc result = dict(row) result[target] = derived output.append(result) return output def _derive_value(operation: str, values: list[Any], *, separator: str) -> Any: if operation == "copy": return values[0] if operation == "upper": return None if values[0] is None else str(values[0]).upper() if operation == "lower": return None if values[0] is None else str(values[0]).lower() if operation == "trim": return None if values[0] is None else str(values[0]).strip() if operation == "concat": return separator.join(str(value) for value in values if value is not None) if operation == "coalesce": return next((value for value in values if value not in (None, "")), None) if any(value is None for value in values): return None left, right = values if isinstance(left, bool) or isinstance(right, bool): raise TypeError("boolean values are not numeric inputs") if operation == "add": return left + right if operation == "subtract": return left - right if operation == "multiply": return left * right if operation == "divide": return left / right raise ValueError(f"unknown derive operation {operation!r}") def _expression_filter_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: parsed = parse_expression(str(config["expression"])) result: list[dict[str, Any]] = [] for row in rows: try: if bool(evaluate_expression(parsed, row)): result.append(dict(row)) except (ArithmeticError, TypeError, ValueError) as exc: raise PipelineExecutionError( f"Cannot evaluate filter expression: {exc}", node_id=node_id, ) from exc return result def _expression_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: target = str(config["target_column"]) parsed = parse_expression(str(config["expression"])) output: list[dict[str, Any]] = [] for row in rows: item = dict(row) try: item[target] = evaluate_expression(parsed, row) except (ArithmeticError, TypeError, ValueError) as exc: raise PipelineExecutionError( f"Cannot evaluate expression for {target!r}: {exc}", node_id=node_id, ) from exc output.append(item) return output def _convert_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: source = str(config["source_column"]) target = str(config["target_column"]) target_type = str(config["target_type"]) on_error = str(config.get("on_error", "fail")) output: list[dict[str, Any]] = [] for row in rows: item = dict(row) try: item[target] = convert_value( row.get(source), target_type, on_error=on_error, # type: ignore[arg-type] ) except (ArithmeticError, TypeError, ValueError) as exc: raise PipelineExecutionError( f"Cannot convert {source!r} to {target_type}: {exc}", node_id=node_id, ) from exc output.append(item) return output def _replace_rows( rows: list[dict[str, Any]], config: dict[str, Any], ) -> list[dict[str, Any]]: source = str(config["source_column"]) target = str(config["target_column"]) mode = str(config.get("mode", "exact")) find = config.get("find") replacement = config.get("replacement") output: list[dict[str, Any]] = [] for row in rows: item = dict(row) value = row.get(source) if mode == "text" and value is not None: item[target] = str(value).replace(str(find), str(replacement)) else: item[target] = replacement if value == find else value output.append(item) return output def _quality_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: rules = config.get("rules", []) action = str(config.get("action", "annotate")) unique_values: dict[str, set[object]] = defaultdict(set) output: list[dict[str, Any]] = [] invalid_count = 0 for row_index, row in enumerate(rows, start=1): failures: list[str] = [] for rule in rules: if not isinstance(rule, dict): continue rule_id = str(rule.get("id") or rule.get("operator") or "rule") column = str(rule.get("column") or "") value = row.get(column) operator = str(rule.get("operator") or "") valid = _quality_rule_matches( value, operator=operator, rule=rule, seen=unique_values[rule_id], ) if not valid: failures.append(rule_id) if failures: invalid_count += 1 if action == "fail": raise PipelineExecutionError( f"Quality rules failed for row {row_index}: {', '.join(failures)}.", node_id=node_id, ) if action == "drop": continue item = dict(row) if action == "annotate": item["_quality_valid"] = not failures item["_quality_errors"] = failures output.append(item) if invalid_count and action not in {"annotate", "drop", "fail"}: raise PipelineExecutionError( f"Unsupported quality action {action!r}.", node_id=node_id, ) return output def _quality_rule_matches( value: Any, *, operator: str, rule: dict[str, Any], seen: set[object], ) -> bool: if operator == "not_null": return value is not None and value != "" if operator == "type": expected = str(rule.get("value") or "") mapping = { "string": str, "integer": int, "number": (int, float, Decimal), "boolean": bool, } expected_type = mapping.get(expected) return expected_type is not None and isinstance(value, expected_type) if operator == "min": return value is not None and value >= rule.get("value") if operator == "max": return value is not None and value <= rule.get("value") if operator == "allowed": allowed = rule.get("values") return isinstance(allowed, list) and value in allowed if operator == "unique": marker = _hashable(value) if marker in seen: return False seen.add(marker) return True return False def _reconcile_rows( left_rows: list[dict[str, Any]], right_rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: left_keys = [str(item) for item in config["left_keys"]] right_keys = [str(item) for item in config["right_keys"]] right_prefix = str(config.get("right_prefix", "observed_")) comparisons = _comparison_fields(config.get("compare_columns")) left_index = _unique_row_index(left_rows, left_keys, node_id=node_id) right_index = _unique_row_index(right_rows, right_keys, node_id=node_id) output: list[dict[str, Any]] = [] for key in dict.fromkeys((*left_index, *right_index)): left = left_index.get(key) right = right_index.get(key) item = dict(left or {}) if right is not None: item.update({f"{right_prefix}{name}": value for name, value in right.items()}) if left is None: status = "missing_expected" differences: list[str] = [] elif right is None: status = "missing_observed" differences = [] else: fields = comparisons or tuple((name, name) for name in left if name not in left_keys) differences = [ left_name for left_name, right_name in fields if left.get(left_name) != right.get(right_name) ] status = "changed" if differences else "match" item["_reconciliation_status"] = status item["_reconciliation_differences"] = differences output.append(item) return output def _comparison_fields(value: object) -> tuple[tuple[str, str], ...]: if not isinstance(value, list): return () fields: list[tuple[str, str]] = [] for item in value: if isinstance(item, str) and item: fields.append((item, item)) elif isinstance(item, dict): left = str(item.get("left") or item.get("column") or "") right = str(item.get("right") or left) if left and right: fields.append((left, right)) return tuple(fields) def _unique_row_index( rows: list[dict[str, Any]], columns: list[str], *, node_id: str, ) -> dict[tuple[Any, ...], dict[str, Any]]: index: dict[tuple[Any, ...], dict[str, Any]] = {} for row in rows: key = tuple(_hashable(row.get(column)) for column in columns) if key in index: raise PipelineExecutionError( f"Reconciliation keys are not unique: {key!r}.", node_id=node_id, ) index[key] = row return index def _execute_subflow( rows: list[dict[str, Any]], config: dict[str, Any], *, source_resolver: SourceResolver | None, execution_depth: int, ) -> PipelineExecutionResult: parameters = config.get("parameters") graph_payload = substitute_parameters( config.get("graph"), parameters if isinstance(parameters, dict) else {}, ) graph = PipelineGraph.model_validate(graph_payload) input_nodes = [ node for node in graph.nodes if node.type == "source.inline" and node.config.get("input_binding") is True ] if len(input_nodes) != 1: raise PipelineExecutionError( "A reusable subflow needs exactly one inline source with input_binding=true." ) input_node = input_nodes[0] graph = graph.model_copy( update={ "nodes": [ ( node.model_copy( update={ "config": { **node.config, "rows": [dict(row) for row in rows], } }, deep=True, ) if node.id == input_node.id else node ) for node in graph.nodes ] }, deep=True, ) return execute_preview( graph, row_limit=MAX_INTERMEDIATE_ROWS, source_resolver=source_resolver, _execution_depth=execution_depth + 1, ) def _join_key(row: dict[str, Any], columns: list[str]) -> tuple[Any, ...] | None: values = tuple(row.get(column) for column in columns) if any(value is None for value in values): return None try: hash(values) except TypeError: return tuple( json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) for value in values ) return values def _ordered_columns(rows: list[dict[str, Any]]) -> list[str]: columns: list[str] = [] for row in rows: for column in row: if column not in columns: columns.append(column) return columns def _merge_join_rows( left_row: dict[str, Any] | None, right_row: dict[str, Any] | None, *, left_columns: list[str], right_columns: list[str], right_prefix: str, ) -> dict[str, Any]: output = { column: left_row.get(column) if left_row is not None else None for column in left_columns } for column in right_columns: output_column = f"{right_prefix}{column}" output[output_column] = right_row.get(column) if right_row is not None else None return output def _guard_intermediate_size(rows: list[dict[str, Any]], *, node_id: str) -> None: if len(rows) > MAX_INTERMEDIATE_ROWS: raise PipelineExecutionError( f"Preview join exceeded the {MAX_INTERMEDIATE_ROWS:,}-row intermediate limit.", node_id=node_id, ) def _filter_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: column = str(config["column"]) operator = str(config["operator"]) expected = config.get("value") result: list[dict[str, Any]] = [] for row in rows: actual = row.get(column) try: matches = _compare(actual, operator, expected) except (TypeError, ValueError) as exc: raise PipelineExecutionError( f"Cannot apply {operator!r} to column {column!r}: {exc}", node_id=node_id, ) from exc if matches: result.append(dict(row)) return result def _compare(actual: Any, operator: str, expected: Any) -> bool: if operator == "is_null": return actual is None if operator == "not_null": return actual is not None if operator == "eq": return actual == expected if operator == "ne": return actual != expected if operator == "contains": return expected is not None and str(expected).casefold() in str(actual or "").casefold() if actual is None or expected is None: return False if operator == "gt": return actual > expected if operator == "gte": return actual >= expected if operator == "lt": return actual < expected if operator == "lte": return actual <= expected raise ValueError(f"unknown operator {operator!r}") def _select_rows(rows: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]: fields = config["fields"] normalized = [ ( field if isinstance(field, str) else str(field["column"]), field if isinstance(field, str) else str(field.get("alias") or field["column"]), ) for field in fields ] return [ {alias: row.get(column) for column, alias in normalized} for row in rows ] def _aggregate_rows( rows: list[dict[str, Any]], config: dict[str, Any], *, node_id: str, ) -> list[dict[str, Any]]: group_by = [str(item) for item in config.get("group_by", [])] aggregates = list(config["aggregates"]) grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list) if rows: for row in rows: grouped[tuple(row.get(column) for column in group_by)].append(row) elif not group_by: grouped[()] = [] output: list[dict[str, Any]] = [] for key, group_rows in grouped.items(): result = {column: value for column, value in zip(group_by, key, strict=True)} for aggregate in aggregates: function = str(aggregate["function"]) column = aggregate.get("column") alias = str(aggregate["alias"]) values = [row.get(column) for row in group_rows if column is not None and row.get(column) is not None] try: if function == "count": result[alias] = len(group_rows) if column in (None, "", "*") else len(values) elif function == "sum": result[alias] = sum(values) if values else 0 elif function == "avg": result[alias] = sum(values) / len(values) if values else None elif function == "min": result[alias] = min(values) if values else None elif function == "max": result[alias] = max(values) if values else None except (ArithmeticError, TypeError, ValueError) as exc: raise PipelineExecutionError( f"Cannot calculate {function.upper()} for {column!r}: {exc}", node_id=node_id, ) from exc output.append(result) return output def _sort_rows(rows: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]: result = [dict(row) for row in rows] for field in reversed(config["fields"]): column = str(field["column"]) reverse = field.get("direction", "asc") == "desc" concrete = [row for row in result if row.get(column) is not None] nulls = [row for row in result if row.get(column) is None] concrete.sort(key=lambda row: _sortable_value(row[column]), reverse=reverse) result = [*concrete, *nulls] return result def _sortable_value(value: Any) -> tuple[str, Any]: if isinstance(value, (int, float, Decimal, str)): return type(value).__name__, value return type(value).__name__, str(value) def _hashable(value: Any) -> Any: if isinstance(value, (dict, list, tuple, set)): return json.dumps(value, sort_keys=True, default=str) return value def _rows_fingerprint(rows: list[dict[str, Any]]) -> str: encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() def _type_name(value: Any) -> str: if value is None: return "unknown" if isinstance(value, bool): return "boolean" if isinstance(value, int): return "integer" if isinstance(value, (float, Decimal)): return "number" if isinstance(value, str): return "string" if isinstance(value, list): return "array" if isinstance(value, dict): return "object" return type(value).__name__.lower() def _execute_inline_source( context: OperatorExecutionContext, ) -> OperatorExecutionResult: rows = [dict(row) for row in context.node.config.get("rows", [])] return OperatorExecutionResult( rows=rows, input_row_count=len(rows), source_fingerprints=( { "node_id": context.node.id, "source_name": context.node.config.get("source_name"), "kind": "inline", "fingerprint": _rows_fingerprint(rows), "row_count": len(rows), }, ), ) def _execute_reference_source( context: OperatorExecutionContext, ) -> OperatorExecutionResult: if context.source_resolver is None: raise PipelineExecutionError( "Datasource-backed preview requires the Datasources catalogue capability.", node_id=context.node.id, ) resolved = context.source_resolver(context.node, MAX_SOURCE_ROWS) rows = [dict(row) for row in resolved.rows] messages = ( ( f"Source preview used {len(rows):,} of " f"{resolved.total_rows:,} rows." ), ) if resolved.truncated else () return OperatorExecutionResult( rows=rows, messages=messages, input_row_count=len(rows), source_fingerprints=( { "node_id": context.node.id, "source_ref": resolved.source_ref, "source_name": context.node.config.get("source_name"), "kind": "datasource", "provider": resolved.provider, "fingerprint": resolved.fingerprint, "row_count": resolved.total_rows, "preview_rows": len(rows), "truncated": resolved.truncated, }, ), ) def _execute_subflow_node( context: OperatorExecutionContext, ) -> OperatorExecutionResult: nested = _execute_subflow( context.input_rows, context.node.config, source_resolver=context.source_resolver, execution_depth=context.execution_depth, ) return OperatorExecutionResult( rows=nested.rows, messages=tuple( f"Subflow {item.node_id}: {message}" for item in nested.node_diagnostics for message in item.messages ), source_fingerprints=tuple( { **fingerprint, "subflow_node_id": context.node.id, } for fingerprint in nested.source_fingerprints ), ) def _register_executors() -> None: executors = { "source.inline": _execute_inline_source, "source.reference": _execute_reference_source, "combine.union": lambda context: OperatorExecutionResult( rows=_union_rows( [ context.outputs[source_id] for source_id in context.inputs_by_port.get("input", ()) ], context.node.config, ) ), "combine.join": lambda context: OperatorExecutionResult( rows=_join_rows( context.outputs[context.inputs_by_port["left"][0]], context.outputs[context.inputs_by_port["right"][0]], context.node.config, node_id=context.node.id, ) ), "filter": lambda context: OperatorExecutionResult( rows=_filter_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "filter.expression": lambda context: OperatorExecutionResult( rows=_expression_filter_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "distinct": lambda context: OperatorExecutionResult( rows=_distinct_rows(context.input_rows, context.node.config) ), "select": lambda context: OperatorExecutionResult( rows=_select_rows(context.input_rows, context.node.config) ), "derive": lambda context: OperatorExecutionResult( rows=_derive_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "expression": lambda context: OperatorExecutionResult( rows=_expression_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "convert": lambda context: OperatorExecutionResult( rows=_convert_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "replace": lambda context: OperatorExecutionResult( rows=_replace_rows(context.input_rows, context.node.config) ), "aggregate": lambda context: OperatorExecutionResult( rows=_aggregate_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "sort": lambda context: OperatorExecutionResult( rows=_sort_rows(context.input_rows, context.node.config) ), "limit": lambda context: OperatorExecutionResult( rows=context.input_rows[: int(context.node.config["count"])] ), "quality.rules": lambda context: OperatorExecutionResult( rows=_quality_rows( context.input_rows, context.node.config, node_id=context.node.id, ) ), "reconcile.compare": lambda context: OperatorExecutionResult( rows=_reconcile_rows( context.outputs[context.inputs_by_port["left"][0]], context.outputs[context.inputs_by_port["right"][0]], context.node.config, node_id=context.node.id, ) ), "subflow": _execute_subflow_node, "output": lambda context: OperatorExecutionResult( rows=[dict(row) for row in context.input_rows] ), } for node_type, executor in executors.items(): if OPERATOR_REGISTRY.executor(node_type) is None: OPERATOR_REGISTRY.register_executor(node_type, executor) _register_executors() __all__ = [ "EXECUTOR_VERSION", "MAX_SOURCE_ROWS", "PipelineExecutionError", "PipelineExecutionResult", "ResolvedSource", "SourceResolver", "execute_preview", "infer_columns", ]