1130 lines
38 KiB
Python
1130 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from decimal import Decimal
|
|
from typing import Any, Callable
|
|
|
|
from govoplan_dataflow.backend.expressions import (
|
|
convert_value,
|
|
evaluate_expression,
|
|
parse_expression,
|
|
)
|
|
from govoplan_dataflow.backend.graph import graph_inputs_by_port, topological_order, validate_graph
|
|
from govoplan_dataflow.backend.operator_registry import (
|
|
OPERATOR_REGISTRY,
|
|
OperatorExecutionContext,
|
|
OperatorExecutionResult,
|
|
)
|
|
from govoplan_dataflow.backend.schemas import (
|
|
DataflowDiagnostic,
|
|
GraphNode,
|
|
NodePreviewDiagnostic,
|
|
NodePreviewResult,
|
|
PipelineGraph,
|
|
PreviewColumn,
|
|
)
|
|
from govoplan_dataflow.backend.subflows import substitute_parameters
|
|
|
|
|
|
EXECUTOR_VERSION = "dataflow-preview-v2"
|
|
MAX_EXECUTION_SECONDS = 2.0
|
|
MAX_RESULT_BYTES = 1_000_000
|
|
MAX_SOURCE_ROWS = 250
|
|
MAX_INTERMEDIATE_ROWS = 10_000
|
|
|
|
|
|
class PipelineExecutionError(RuntimeError):
|
|
def __init__(
|
|
self,
|
|
message: str,
|
|
*,
|
|
node_id: str | None = None,
|
|
node_diagnostics: tuple[NodePreviewDiagnostic, ...] = (),
|
|
source_fingerprints: tuple[dict[str, Any], ...] = (),
|
|
input_row_count: int = 0,
|
|
diagnostics: tuple[DataflowDiagnostic, ...] = (),
|
|
node_preview: NodePreviewResult | None = None,
|
|
) -> None:
|
|
super().__init__(message)
|
|
self.node_id = node_id
|
|
self.node_diagnostics = node_diagnostics
|
|
self.source_fingerprints = source_fingerprints
|
|
self.input_row_count = input_row_count
|
|
self.diagnostics = diagnostics
|
|
self.node_preview = node_preview
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PipelineExecutionResult:
|
|
rows: list[dict[str, Any]]
|
|
total_rows: int
|
|
truncated: bool
|
|
columns: list[PreviewColumn]
|
|
diagnostics: list[DataflowDiagnostic]
|
|
node_diagnostics: list[NodePreviewDiagnostic]
|
|
node_preview: NodePreviewResult | None
|
|
source_fingerprints: list[dict[str, Any]]
|
|
input_row_count: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResolvedSource:
|
|
rows: tuple[dict[str, Any], ...]
|
|
source_ref: str
|
|
provider: str
|
|
fingerprint: str
|
|
total_rows: int
|
|
truncated: bool = False
|
|
|
|
|
|
SourceResolver = Callable[[GraphNode, int], ResolvedSource]
|
|
|
|
|
|
def execute_preview(
|
|
graph: PipelineGraph,
|
|
*,
|
|
row_limit: int,
|
|
source_resolver: SourceResolver | None = None,
|
|
preview_node_id: str | None = None,
|
|
_execution_depth: int = 0,
|
|
) -> PipelineExecutionResult:
|
|
if _execution_depth > 5:
|
|
raise PipelineExecutionError("Subflows are limited to five nested levels.")
|
|
validation = validate_graph(graph)
|
|
errors = [item for item in validation if item.severity == "error"]
|
|
if errors:
|
|
raise PipelineExecutionError(errors[0].message, node_id=errors[0].node_id)
|
|
|
|
node_by_id = {node.id: node for node in graph.nodes}
|
|
if preview_node_id is not None and preview_node_id not in node_by_id:
|
|
raise PipelineExecutionError(
|
|
"The requested preview node is not part of this pipeline.",
|
|
node_id=preview_node_id,
|
|
)
|
|
inputs = graph_inputs_by_port(graph)
|
|
ordered, cyclic = topological_order(graph)
|
|
if cyclic:
|
|
raise PipelineExecutionError("Pipeline graph contains a cycle")
|
|
|
|
outputs: dict[str, list[dict[str, Any]]] = {}
|
|
node_diagnostics: list[NodePreviewDiagnostic] = []
|
|
source_fingerprints: list[dict[str, Any]] = []
|
|
started = time.monotonic()
|
|
input_row_count = 0
|
|
|
|
for node_id in ordered:
|
|
node = node_by_id[node_id]
|
|
node_started = time.monotonic()
|
|
node_inputs = inputs.get(node_id, {})
|
|
input_sets = [
|
|
outputs[source_id]
|
|
for port_sources in node_inputs.values()
|
|
for source_id in port_sources
|
|
]
|
|
input_rows = input_sets[0] if len(input_sets) == 1 else []
|
|
node_messages: list[str] = []
|
|
if time.monotonic() - started > MAX_EXECUTION_SECONDS:
|
|
failed = NodePreviewDiagnostic(
|
|
node_id=node.id,
|
|
status="failed",
|
|
input_rows=sum(len(rows) for rows in input_sets),
|
|
output_rows=0,
|
|
duration_ms=0,
|
|
columns=[],
|
|
messages=["Preview exceeded the two-second execution limit"],
|
|
)
|
|
raise PipelineExecutionError(
|
|
"Preview exceeded the two-second execution limit",
|
|
node_id=node.id,
|
|
node_diagnostics=tuple([*node_diagnostics, failed]),
|
|
source_fingerprints=tuple(source_fingerprints),
|
|
input_row_count=input_row_count,
|
|
diagnostics=tuple(item for item in validation if item.severity != "error"),
|
|
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
|
)
|
|
try:
|
|
executor = OPERATOR_REGISTRY.executor(node.type)
|
|
if executor is None:
|
|
raise PipelineExecutionError(
|
|
f"Node type {node.type!r} has no registered executor.",
|
|
node_id=node.id,
|
|
)
|
|
execution = executor(
|
|
OperatorExecutionContext(
|
|
node=node,
|
|
inputs_by_port=node_inputs,
|
|
outputs=outputs,
|
|
input_sets=input_sets,
|
|
input_rows=input_rows,
|
|
source_resolver=source_resolver,
|
|
execution_depth=_execution_depth,
|
|
)
|
|
)
|
|
output_rows = execution.rows
|
|
input_row_count += execution.input_row_count
|
|
source_fingerprints.extend(execution.source_fingerprints)
|
|
node_messages.extend(execution.messages)
|
|
if node.type == "source.reference" and execution.messages:
|
|
validation.extend(
|
|
DataflowDiagnostic(
|
|
severity="warning",
|
|
code="source.preview_truncated",
|
|
message=message,
|
|
node_id=node.id,
|
|
)
|
|
for message in execution.messages
|
|
)
|
|
if len(json.dumps(output_rows, default=str).encode("utf-8")) > MAX_RESULT_BYTES:
|
|
raise PipelineExecutionError(
|
|
"A preview node exceeded the one-megabyte result limit.",
|
|
node_id=node.id,
|
|
)
|
|
except (PipelineExecutionError, ArithmeticError, KeyError, TypeError, ValueError) as exc:
|
|
execution_error = (
|
|
exc
|
|
if isinstance(exc, PipelineExecutionError)
|
|
else PipelineExecutionError(str(exc), node_id=node.id)
|
|
)
|
|
failed = NodePreviewDiagnostic(
|
|
node_id=execution_error.node_id or node.id,
|
|
status="failed",
|
|
input_rows=sum(len(rows) for rows in input_sets),
|
|
output_rows=0,
|
|
duration_ms=round((time.monotonic() - node_started) * 1000, 3),
|
|
columns=[],
|
|
messages=[str(execution_error)],
|
|
)
|
|
raise PipelineExecutionError(
|
|
str(execution_error),
|
|
node_id=execution_error.node_id or node.id,
|
|
node_diagnostics=tuple([*node_diagnostics, failed]),
|
|
source_fingerprints=tuple(source_fingerprints),
|
|
input_row_count=input_row_count,
|
|
diagnostics=tuple(item for item in validation if item.severity != "error"),
|
|
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
|
) from exc
|
|
|
|
outputs[node.id] = output_rows
|
|
node_diagnostics.append(
|
|
NodePreviewDiagnostic(
|
|
node_id=node.id,
|
|
status="succeeded",
|
|
input_rows=sum(len(rows) for rows in input_sets),
|
|
output_rows=len(output_rows),
|
|
duration_ms=round((time.monotonic() - node_started) * 1000, 3),
|
|
columns=infer_columns(output_rows),
|
|
messages=node_messages,
|
|
)
|
|
)
|
|
|
|
output_node = next(node for node in graph.nodes if node.type == "output")
|
|
all_rows = outputs[output_node.id]
|
|
rows = all_rows[:row_limit]
|
|
return PipelineExecutionResult(
|
|
rows=rows,
|
|
total_rows=len(all_rows),
|
|
truncated=len(rows) < len(all_rows),
|
|
columns=infer_columns(all_rows),
|
|
diagnostics=[item for item in validation if item.severity != "error"],
|
|
node_diagnostics=node_diagnostics,
|
|
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
|
source_fingerprints=source_fingerprints,
|
|
input_row_count=input_row_count,
|
|
)
|
|
|
|
|
|
def _node_preview(
|
|
outputs: dict[str, list[dict[str, Any]]],
|
|
node_id: str | None,
|
|
row_limit: int,
|
|
) -> NodePreviewResult | None:
|
|
if node_id is None or node_id not in outputs:
|
|
return None
|
|
all_rows = outputs[node_id]
|
|
rows = [dict(row) for row in all_rows[:row_limit]]
|
|
return NodePreviewResult(
|
|
node_id=node_id,
|
|
columns=infer_columns(all_rows),
|
|
rows=rows,
|
|
total_rows=len(all_rows),
|
|
truncated=len(rows) < len(all_rows),
|
|
)
|
|
|
|
|
|
def infer_columns(rows: list[dict[str, Any]]) -> list[PreviewColumn]:
|
|
names: list[str] = []
|
|
for row in rows:
|
|
for name in row:
|
|
if name not in names:
|
|
names.append(name)
|
|
columns: list[PreviewColumn] = []
|
|
for name in names:
|
|
values = [row.get(name) for row in rows]
|
|
concrete = [value for value in values if value is not None]
|
|
inferred = _type_name(concrete[0]) if concrete else "unknown"
|
|
if any(_type_name(value) != inferred for value in concrete[1:]):
|
|
inferred = "mixed"
|
|
columns.append(
|
|
PreviewColumn(
|
|
name=name,
|
|
type=inferred,
|
|
nullable=len(concrete) != len(values),
|
|
)
|
|
)
|
|
return columns
|
|
|
|
|
|
def _union_rows(
|
|
inputs: list[list[dict[str, Any]]],
|
|
config: dict[str, Any],
|
|
) -> list[dict[str, Any]]:
|
|
rows = [dict(row) for input_rows in inputs for row in input_rows]
|
|
if config.get("mode", "all") == "distinct":
|
|
return _distinct_rows(rows, {})
|
|
return rows
|
|
|
|
|
|
def _join_rows(
|
|
left_rows: list[dict[str, Any]],
|
|
right_rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
left_keys = [str(item) for item in config["left_keys"]]
|
|
right_keys = [str(item) for item in config["right_keys"]]
|
|
join_type = str(config.get("join_type", "inner"))
|
|
right_prefix = str(config.get("right_prefix", "right_"))
|
|
left_columns = _ordered_columns(left_rows)
|
|
right_columns = _ordered_columns(right_rows)
|
|
right_index: dict[tuple[Any, ...], list[tuple[int, dict[str, Any]]]] = defaultdict(list)
|
|
for index, row in enumerate(right_rows):
|
|
key = _join_key(row, right_keys)
|
|
if key is not None:
|
|
right_index[key].append((index, row))
|
|
|
|
output: list[dict[str, Any]] = []
|
|
matched_right: set[int] = set()
|
|
for left_row in left_rows:
|
|
key = _join_key(left_row, left_keys)
|
|
matches = right_index.get(key, ()) if key is not None else ()
|
|
if matches:
|
|
for right_index_value, right_row in matches:
|
|
matched_right.add(right_index_value)
|
|
output.append(
|
|
_merge_join_rows(
|
|
left_row,
|
|
right_row,
|
|
left_columns=left_columns,
|
|
right_columns=right_columns,
|
|
right_prefix=right_prefix,
|
|
)
|
|
)
|
|
_guard_intermediate_size(output, node_id=node_id)
|
|
elif join_type in {"left", "full"}:
|
|
output.append(
|
|
_merge_join_rows(
|
|
left_row,
|
|
None,
|
|
left_columns=left_columns,
|
|
right_columns=right_columns,
|
|
right_prefix=right_prefix,
|
|
)
|
|
)
|
|
|
|
if join_type in {"right", "full"}:
|
|
for index, right_row in enumerate(right_rows):
|
|
if index in matched_right:
|
|
continue
|
|
output.append(
|
|
_merge_join_rows(
|
|
None,
|
|
right_row,
|
|
left_columns=left_columns,
|
|
right_columns=right_columns,
|
|
right_prefix=right_prefix,
|
|
)
|
|
)
|
|
_guard_intermediate_size(output, node_id=node_id)
|
|
return output
|
|
|
|
|
|
def _distinct_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
) -> list[dict[str, Any]]:
|
|
columns = [str(item) for item in config.get("columns", [])]
|
|
seen: set[str] = set()
|
|
output: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
value = {column: row.get(column) for column in columns} if columns else row
|
|
identity = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
|
if identity in seen:
|
|
continue
|
|
seen.add(identity)
|
|
output.append(dict(row))
|
|
return output
|
|
|
|
|
|
def _derive_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
target = str(config["target_column"])
|
|
operation = str(config["operation"])
|
|
columns = [str(item) for item in config["source_columns"]]
|
|
separator = str(config.get("separator", " "))
|
|
output: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
values = [row.get(column) for column in columns]
|
|
try:
|
|
derived = _derive_value(operation, values, separator=separator)
|
|
except (ArithmeticError, TypeError, ValueError) as exc:
|
|
raise PipelineExecutionError(
|
|
f"Cannot derive {target!r} with {operation!r}: {exc}",
|
|
node_id=node_id,
|
|
) from exc
|
|
result = dict(row)
|
|
result[target] = derived
|
|
output.append(result)
|
|
return output
|
|
|
|
|
|
def _derive_value(operation: str, values: list[Any], *, separator: str) -> Any:
|
|
if operation == "copy":
|
|
return values[0]
|
|
if operation == "upper":
|
|
return None if values[0] is None else str(values[0]).upper()
|
|
if operation == "lower":
|
|
return None if values[0] is None else str(values[0]).lower()
|
|
if operation == "trim":
|
|
return None if values[0] is None else str(values[0]).strip()
|
|
if operation == "concat":
|
|
return separator.join(str(value) for value in values if value is not None)
|
|
if operation == "coalesce":
|
|
return next((value for value in values if value not in (None, "")), None)
|
|
if any(value is None for value in values):
|
|
return None
|
|
left, right = values
|
|
if isinstance(left, bool) or isinstance(right, bool):
|
|
raise TypeError("boolean values are not numeric inputs")
|
|
if operation == "add":
|
|
return left + right
|
|
if operation == "subtract":
|
|
return left - right
|
|
if operation == "multiply":
|
|
return left * right
|
|
if operation == "divide":
|
|
return left / right
|
|
raise ValueError(f"unknown derive operation {operation!r}")
|
|
|
|
|
|
def _expression_filter_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
parsed = parse_expression(str(config["expression"]))
|
|
result: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
try:
|
|
if bool(evaluate_expression(parsed, row)):
|
|
result.append(dict(row))
|
|
except (ArithmeticError, TypeError, ValueError) as exc:
|
|
raise PipelineExecutionError(
|
|
f"Cannot evaluate filter expression: {exc}",
|
|
node_id=node_id,
|
|
) from exc
|
|
return result
|
|
|
|
|
|
def _expression_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
target = str(config["target_column"])
|
|
parsed = parse_expression(str(config["expression"]))
|
|
output: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
try:
|
|
item[target] = evaluate_expression(parsed, row)
|
|
except (ArithmeticError, TypeError, ValueError) as exc:
|
|
raise PipelineExecutionError(
|
|
f"Cannot evaluate expression for {target!r}: {exc}",
|
|
node_id=node_id,
|
|
) from exc
|
|
output.append(item)
|
|
return output
|
|
|
|
|
|
def _convert_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
source = str(config["source_column"])
|
|
target = str(config["target_column"])
|
|
target_type = str(config["target_type"])
|
|
on_error = str(config.get("on_error", "fail"))
|
|
output: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
try:
|
|
item[target] = convert_value(
|
|
row.get(source),
|
|
target_type,
|
|
on_error=on_error, # type: ignore[arg-type]
|
|
)
|
|
except (ArithmeticError, TypeError, ValueError) as exc:
|
|
raise PipelineExecutionError(
|
|
f"Cannot convert {source!r} to {target_type}: {exc}",
|
|
node_id=node_id,
|
|
) from exc
|
|
output.append(item)
|
|
return output
|
|
|
|
|
|
def _replace_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
) -> list[dict[str, Any]]:
|
|
source = str(config["source_column"])
|
|
target = str(config["target_column"])
|
|
mode = str(config.get("mode", "exact"))
|
|
find = config.get("find")
|
|
replacement = config.get("replacement")
|
|
output: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
value = row.get(source)
|
|
if mode == "text" and value is not None:
|
|
item[target] = str(value).replace(str(find), str(replacement))
|
|
else:
|
|
item[target] = replacement if value == find else value
|
|
output.append(item)
|
|
return output
|
|
|
|
|
|
def _quality_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
rules = config.get("rules", [])
|
|
action = str(config.get("action", "annotate"))
|
|
unique_values: dict[str, set[object]] = defaultdict(set)
|
|
output: list[dict[str, Any]] = []
|
|
invalid_count = 0
|
|
for row_index, row in enumerate(rows, start=1):
|
|
failures: list[str] = []
|
|
for rule in rules:
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
rule_id = str(rule.get("id") or rule.get("operator") or "rule")
|
|
column = str(rule.get("column") or "")
|
|
value = row.get(column)
|
|
operator = str(rule.get("operator") or "")
|
|
valid = _quality_rule_matches(
|
|
value,
|
|
operator=operator,
|
|
rule=rule,
|
|
seen=unique_values[rule_id],
|
|
)
|
|
if not valid:
|
|
failures.append(rule_id)
|
|
if failures:
|
|
invalid_count += 1
|
|
if action == "fail":
|
|
raise PipelineExecutionError(
|
|
f"Quality rules failed for row {row_index}: {', '.join(failures)}.",
|
|
node_id=node_id,
|
|
)
|
|
if action == "drop":
|
|
continue
|
|
item = dict(row)
|
|
if action == "annotate":
|
|
item["_quality_valid"] = not failures
|
|
item["_quality_errors"] = failures
|
|
output.append(item)
|
|
if invalid_count and action not in {"annotate", "drop", "fail"}:
|
|
raise PipelineExecutionError(
|
|
f"Unsupported quality action {action!r}.",
|
|
node_id=node_id,
|
|
)
|
|
return output
|
|
|
|
|
|
def _quality_rule_matches(
|
|
value: Any,
|
|
*,
|
|
operator: str,
|
|
rule: dict[str, Any],
|
|
seen: set[object],
|
|
) -> bool:
|
|
if operator == "not_null":
|
|
return value is not None and value != ""
|
|
if operator == "type":
|
|
expected = str(rule.get("value") or "")
|
|
mapping = {
|
|
"string": str,
|
|
"integer": int,
|
|
"number": (int, float, Decimal),
|
|
"boolean": bool,
|
|
}
|
|
expected_type = mapping.get(expected)
|
|
return expected_type is not None and isinstance(value, expected_type)
|
|
if operator == "min":
|
|
return value is not None and value >= rule.get("value")
|
|
if operator == "max":
|
|
return value is not None and value <= rule.get("value")
|
|
if operator == "allowed":
|
|
allowed = rule.get("values")
|
|
return isinstance(allowed, list) and value in allowed
|
|
if operator == "unique":
|
|
marker = _hashable(value)
|
|
if marker in seen:
|
|
return False
|
|
seen.add(marker)
|
|
return True
|
|
return False
|
|
|
|
|
|
def _reconcile_rows(
|
|
left_rows: list[dict[str, Any]],
|
|
right_rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
left_keys = [str(item) for item in config["left_keys"]]
|
|
right_keys = [str(item) for item in config["right_keys"]]
|
|
right_prefix = str(config.get("right_prefix", "observed_"))
|
|
comparisons = _comparison_fields(config.get("compare_columns"))
|
|
left_index = _unique_row_index(left_rows, left_keys, node_id=node_id)
|
|
right_index = _unique_row_index(right_rows, right_keys, node_id=node_id)
|
|
output: list[dict[str, Any]] = []
|
|
for key in dict.fromkeys((*left_index, *right_index)):
|
|
left = left_index.get(key)
|
|
right = right_index.get(key)
|
|
item = dict(left or {})
|
|
if right is not None:
|
|
item.update({f"{right_prefix}{name}": value for name, value in right.items()})
|
|
if left is None:
|
|
status = "missing_expected"
|
|
differences: list[str] = []
|
|
elif right is None:
|
|
status = "missing_observed"
|
|
differences = []
|
|
else:
|
|
fields = comparisons or tuple((name, name) for name in left if name not in left_keys)
|
|
differences = [
|
|
left_name
|
|
for left_name, right_name in fields
|
|
if left.get(left_name) != right.get(right_name)
|
|
]
|
|
status = "changed" if differences else "match"
|
|
item["_reconciliation_status"] = status
|
|
item["_reconciliation_differences"] = differences
|
|
output.append(item)
|
|
return output
|
|
|
|
|
|
def _comparison_fields(value: object) -> tuple[tuple[str, str], ...]:
|
|
if not isinstance(value, list):
|
|
return ()
|
|
fields: list[tuple[str, str]] = []
|
|
for item in value:
|
|
if isinstance(item, str) and item:
|
|
fields.append((item, item))
|
|
elif isinstance(item, dict):
|
|
left = str(item.get("left") or item.get("column") or "")
|
|
right = str(item.get("right") or left)
|
|
if left and right:
|
|
fields.append((left, right))
|
|
return tuple(fields)
|
|
|
|
|
|
def _unique_row_index(
|
|
rows: list[dict[str, Any]],
|
|
columns: list[str],
|
|
*,
|
|
node_id: str,
|
|
) -> dict[tuple[Any, ...], dict[str, Any]]:
|
|
index: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
for row in rows:
|
|
key = tuple(_hashable(row.get(column)) for column in columns)
|
|
if key in index:
|
|
raise PipelineExecutionError(
|
|
f"Reconciliation keys are not unique: {key!r}.",
|
|
node_id=node_id,
|
|
)
|
|
index[key] = row
|
|
return index
|
|
|
|
|
|
def _execute_subflow(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
source_resolver: SourceResolver | None,
|
|
execution_depth: int,
|
|
) -> PipelineExecutionResult:
|
|
parameters = config.get("parameters")
|
|
graph_payload = substitute_parameters(
|
|
config.get("graph"),
|
|
parameters if isinstance(parameters, dict) else {},
|
|
)
|
|
graph = PipelineGraph.model_validate(graph_payload)
|
|
input_nodes = [
|
|
node
|
|
for node in graph.nodes
|
|
if node.type == "source.inline" and node.config.get("input_binding") is True
|
|
]
|
|
if len(input_nodes) != 1:
|
|
raise PipelineExecutionError(
|
|
"A reusable subflow needs exactly one inline source with input_binding=true."
|
|
)
|
|
input_node = input_nodes[0]
|
|
graph = graph.model_copy(
|
|
update={
|
|
"nodes": [
|
|
(
|
|
node.model_copy(
|
|
update={
|
|
"config": {
|
|
**node.config,
|
|
"rows": [dict(row) for row in rows],
|
|
}
|
|
},
|
|
deep=True,
|
|
)
|
|
if node.id == input_node.id
|
|
else node
|
|
)
|
|
for node in graph.nodes
|
|
]
|
|
},
|
|
deep=True,
|
|
)
|
|
return execute_preview(
|
|
graph,
|
|
row_limit=MAX_INTERMEDIATE_ROWS,
|
|
source_resolver=source_resolver,
|
|
_execution_depth=execution_depth + 1,
|
|
)
|
|
|
|
|
|
|
|
|
|
def _join_key(row: dict[str, Any], columns: list[str]) -> tuple[Any, ...] | None:
|
|
values = tuple(row.get(column) for column in columns)
|
|
if any(value is None for value in values):
|
|
return None
|
|
try:
|
|
hash(values)
|
|
except TypeError:
|
|
return tuple(
|
|
json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
|
for value in values
|
|
)
|
|
return values
|
|
|
|
|
|
def _ordered_columns(rows: list[dict[str, Any]]) -> list[str]:
|
|
columns: list[str] = []
|
|
for row in rows:
|
|
for column in row:
|
|
if column not in columns:
|
|
columns.append(column)
|
|
return columns
|
|
|
|
|
|
def _merge_join_rows(
|
|
left_row: dict[str, Any] | None,
|
|
right_row: dict[str, Any] | None,
|
|
*,
|
|
left_columns: list[str],
|
|
right_columns: list[str],
|
|
right_prefix: str,
|
|
) -> dict[str, Any]:
|
|
output = {
|
|
column: left_row.get(column) if left_row is not None else None
|
|
for column in left_columns
|
|
}
|
|
for column in right_columns:
|
|
output_column = f"{right_prefix}{column}"
|
|
output[output_column] = right_row.get(column) if right_row is not None else None
|
|
return output
|
|
|
|
|
|
def _guard_intermediate_size(rows: list[dict[str, Any]], *, node_id: str) -> None:
|
|
if len(rows) > MAX_INTERMEDIATE_ROWS:
|
|
raise PipelineExecutionError(
|
|
f"Preview join exceeded the {MAX_INTERMEDIATE_ROWS:,}-row intermediate limit.",
|
|
node_id=node_id,
|
|
)
|
|
|
|
|
|
def _filter_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
column = str(config["column"])
|
|
operator = str(config["operator"])
|
|
expected = config.get("value")
|
|
result: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
actual = row.get(column)
|
|
try:
|
|
matches = _compare(actual, operator, expected)
|
|
except (TypeError, ValueError) as exc:
|
|
raise PipelineExecutionError(
|
|
f"Cannot apply {operator!r} to column {column!r}: {exc}",
|
|
node_id=node_id,
|
|
) from exc
|
|
if matches:
|
|
result.append(dict(row))
|
|
return result
|
|
|
|
|
|
def _compare(actual: Any, operator: str, expected: Any) -> bool:
|
|
if operator == "is_null":
|
|
return actual is None
|
|
if operator == "not_null":
|
|
return actual is not None
|
|
if operator == "eq":
|
|
return actual == expected
|
|
if operator == "ne":
|
|
return actual != expected
|
|
if operator == "contains":
|
|
return expected is not None and str(expected).casefold() in str(actual or "").casefold()
|
|
if actual is None or expected is None:
|
|
return False
|
|
if operator == "gt":
|
|
return actual > expected
|
|
if operator == "gte":
|
|
return actual >= expected
|
|
if operator == "lt":
|
|
return actual < expected
|
|
if operator == "lte":
|
|
return actual <= expected
|
|
raise ValueError(f"unknown operator {operator!r}")
|
|
|
|
|
|
def _select_rows(rows: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]:
|
|
fields = config["fields"]
|
|
normalized = [
|
|
(
|
|
field if isinstance(field, str) else str(field["column"]),
|
|
field if isinstance(field, str) else str(field.get("alias") or field["column"]),
|
|
)
|
|
for field in fields
|
|
]
|
|
return [
|
|
{alias: row.get(column) for column, alias in normalized}
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def _aggregate_rows(
|
|
rows: list[dict[str, Any]],
|
|
config: dict[str, Any],
|
|
*,
|
|
node_id: str,
|
|
) -> list[dict[str, Any]]:
|
|
group_by = [str(item) for item in config.get("group_by", [])]
|
|
aggregates = list(config["aggregates"])
|
|
grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list)
|
|
if rows:
|
|
for row in rows:
|
|
grouped[tuple(row.get(column) for column in group_by)].append(row)
|
|
elif not group_by:
|
|
grouped[()] = []
|
|
|
|
output: list[dict[str, Any]] = []
|
|
for key, group_rows in grouped.items():
|
|
result = {column: value for column, value in zip(group_by, key, strict=True)}
|
|
for aggregate in aggregates:
|
|
function = str(aggregate["function"])
|
|
column = aggregate.get("column")
|
|
alias = str(aggregate["alias"])
|
|
values = [row.get(column) for row in group_rows if column is not None and row.get(column) is not None]
|
|
try:
|
|
if function == "count":
|
|
result[alias] = len(group_rows) if column in (None, "", "*") else len(values)
|
|
elif function == "sum":
|
|
result[alias] = sum(values) if values else 0
|
|
elif function == "avg":
|
|
result[alias] = sum(values) / len(values) if values else None
|
|
elif function == "min":
|
|
result[alias] = min(values) if values else None
|
|
elif function == "max":
|
|
result[alias] = max(values) if values else None
|
|
except (ArithmeticError, TypeError, ValueError) as exc:
|
|
raise PipelineExecutionError(
|
|
f"Cannot calculate {function.upper()} for {column!r}: {exc}",
|
|
node_id=node_id,
|
|
) from exc
|
|
output.append(result)
|
|
return output
|
|
|
|
|
|
def _sort_rows(rows: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]:
|
|
result = [dict(row) for row in rows]
|
|
for field in reversed(config["fields"]):
|
|
column = str(field["column"])
|
|
reverse = field.get("direction", "asc") == "desc"
|
|
concrete = [row for row in result if row.get(column) is not None]
|
|
nulls = [row for row in result if row.get(column) is None]
|
|
concrete.sort(key=lambda row: _sortable_value(row[column]), reverse=reverse)
|
|
result = [*concrete, *nulls]
|
|
return result
|
|
|
|
|
|
def _sortable_value(value: Any) -> tuple[str, Any]:
|
|
if isinstance(value, (int, float, Decimal, str)):
|
|
return type(value).__name__, value
|
|
return type(value).__name__, str(value)
|
|
|
|
|
|
def _hashable(value: Any) -> Any:
|
|
if isinstance(value, (dict, list, tuple, set)):
|
|
return json.dumps(value, sort_keys=True, default=str)
|
|
return value
|
|
|
|
|
|
def _rows_fingerprint(rows: list[dict[str, Any]]) -> str:
|
|
encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str)
|
|
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _type_name(value: Any) -> str:
|
|
if value is None:
|
|
return "unknown"
|
|
if isinstance(value, bool):
|
|
return "boolean"
|
|
if isinstance(value, int):
|
|
return "integer"
|
|
if isinstance(value, (float, Decimal)):
|
|
return "number"
|
|
if isinstance(value, str):
|
|
return "string"
|
|
if isinstance(value, list):
|
|
return "array"
|
|
if isinstance(value, dict):
|
|
return "object"
|
|
return type(value).__name__.lower()
|
|
|
|
|
|
def _execute_inline_source(
|
|
context: OperatorExecutionContext,
|
|
) -> OperatorExecutionResult:
|
|
rows = [dict(row) for row in context.node.config.get("rows", [])]
|
|
return OperatorExecutionResult(
|
|
rows=rows,
|
|
input_row_count=len(rows),
|
|
source_fingerprints=(
|
|
{
|
|
"node_id": context.node.id,
|
|
"source_name": context.node.config.get("source_name"),
|
|
"kind": "inline",
|
|
"fingerprint": _rows_fingerprint(rows),
|
|
"row_count": len(rows),
|
|
},
|
|
),
|
|
)
|
|
|
|
|
|
def _execute_reference_source(
|
|
context: OperatorExecutionContext,
|
|
) -> OperatorExecutionResult:
|
|
if context.source_resolver is None:
|
|
raise PipelineExecutionError(
|
|
"Datasource-backed preview requires the Datasources catalogue capability.",
|
|
node_id=context.node.id,
|
|
)
|
|
resolved = context.source_resolver(context.node, MAX_SOURCE_ROWS)
|
|
rows = [dict(row) for row in resolved.rows]
|
|
messages = (
|
|
(
|
|
f"Source preview used {len(rows):,} of "
|
|
f"{resolved.total_rows:,} rows."
|
|
),
|
|
) if resolved.truncated else ()
|
|
return OperatorExecutionResult(
|
|
rows=rows,
|
|
messages=messages,
|
|
input_row_count=len(rows),
|
|
source_fingerprints=(
|
|
{
|
|
"node_id": context.node.id,
|
|
"source_ref": resolved.source_ref,
|
|
"source_name": context.node.config.get("source_name"),
|
|
"kind": "datasource",
|
|
"provider": resolved.provider,
|
|
"fingerprint": resolved.fingerprint,
|
|
"row_count": resolved.total_rows,
|
|
"preview_rows": len(rows),
|
|
"truncated": resolved.truncated,
|
|
},
|
|
),
|
|
)
|
|
|
|
|
|
def _execute_subflow_node(
|
|
context: OperatorExecutionContext,
|
|
) -> OperatorExecutionResult:
|
|
nested = _execute_subflow(
|
|
context.input_rows,
|
|
context.node.config,
|
|
source_resolver=context.source_resolver,
|
|
execution_depth=context.execution_depth,
|
|
)
|
|
return OperatorExecutionResult(
|
|
rows=nested.rows,
|
|
messages=tuple(
|
|
f"Subflow {item.node_id}: {message}"
|
|
for item in nested.node_diagnostics
|
|
for message in item.messages
|
|
),
|
|
source_fingerprints=tuple(
|
|
{
|
|
**fingerprint,
|
|
"subflow_node_id": context.node.id,
|
|
}
|
|
for fingerprint in nested.source_fingerprints
|
|
),
|
|
)
|
|
|
|
|
|
def _register_executors() -> None:
|
|
executors = {
|
|
"source.inline": _execute_inline_source,
|
|
"source.reference": _execute_reference_source,
|
|
"combine.union": lambda context: OperatorExecutionResult(
|
|
rows=_union_rows(
|
|
[
|
|
context.outputs[source_id]
|
|
for source_id in context.inputs_by_port.get("input", ())
|
|
],
|
|
context.node.config,
|
|
)
|
|
),
|
|
"combine.join": lambda context: OperatorExecutionResult(
|
|
rows=_join_rows(
|
|
context.outputs[context.inputs_by_port["left"][0]],
|
|
context.outputs[context.inputs_by_port["right"][0]],
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"filter": lambda context: OperatorExecutionResult(
|
|
rows=_filter_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"filter.expression": lambda context: OperatorExecutionResult(
|
|
rows=_expression_filter_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"distinct": lambda context: OperatorExecutionResult(
|
|
rows=_distinct_rows(context.input_rows, context.node.config)
|
|
),
|
|
"select": lambda context: OperatorExecutionResult(
|
|
rows=_select_rows(context.input_rows, context.node.config)
|
|
),
|
|
"derive": lambda context: OperatorExecutionResult(
|
|
rows=_derive_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"expression": lambda context: OperatorExecutionResult(
|
|
rows=_expression_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"convert": lambda context: OperatorExecutionResult(
|
|
rows=_convert_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"replace": lambda context: OperatorExecutionResult(
|
|
rows=_replace_rows(context.input_rows, context.node.config)
|
|
),
|
|
"aggregate": lambda context: OperatorExecutionResult(
|
|
rows=_aggregate_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"sort": lambda context: OperatorExecutionResult(
|
|
rows=_sort_rows(context.input_rows, context.node.config)
|
|
),
|
|
"limit": lambda context: OperatorExecutionResult(
|
|
rows=context.input_rows[: int(context.node.config["count"])]
|
|
),
|
|
"quality.rules": lambda context: OperatorExecutionResult(
|
|
rows=_quality_rows(
|
|
context.input_rows,
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"reconcile.compare": lambda context: OperatorExecutionResult(
|
|
rows=_reconcile_rows(
|
|
context.outputs[context.inputs_by_port["left"][0]],
|
|
context.outputs[context.inputs_by_port["right"][0]],
|
|
context.node.config,
|
|
node_id=context.node.id,
|
|
)
|
|
),
|
|
"subflow": _execute_subflow_node,
|
|
"output": lambda context: OperatorExecutionResult(
|
|
rows=[dict(row) for row in context.input_rows]
|
|
),
|
|
}
|
|
for node_type, executor in executors.items():
|
|
if OPERATOR_REGISTRY.executor(node_type) is None:
|
|
OPERATOR_REGISTRY.register_executor(node_type, executor)
|
|
|
|
|
|
_register_executors()
|
|
|
|
|
|
__all__ = [
|
|
"EXECUTOR_VERSION",
|
|
"MAX_SOURCE_ROWS",
|
|
"PipelineExecutionError",
|
|
"PipelineExecutionResult",
|
|
"ResolvedSource",
|
|
"SourceResolver",
|
|
"execute_preview",
|
|
"infer_columns",
|
|
]
|