feat: add extensible operators and golden flows
This commit is contained in:
@@ -8,7 +8,17 @@ from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
from typing import Any, Callable
|
||||
|
||||
from govoplan_dataflow.backend.expressions import (
|
||||
convert_value,
|
||||
evaluate_expression,
|
||||
parse_expression,
|
||||
)
|
||||
from govoplan_dataflow.backend.graph import graph_inputs_by_port, topological_order, validate_graph
|
||||
from govoplan_dataflow.backend.operator_registry import (
|
||||
OPERATOR_REGISTRY,
|
||||
OperatorExecutionContext,
|
||||
OperatorExecutionResult,
|
||||
)
|
||||
from govoplan_dataflow.backend.schemas import (
|
||||
DataflowDiagnostic,
|
||||
GraphNode,
|
||||
@@ -17,6 +27,7 @@ from govoplan_dataflow.backend.schemas import (
|
||||
PipelineGraph,
|
||||
PreviewColumn,
|
||||
)
|
||||
from govoplan_dataflow.backend.subflows import substitute_parameters
|
||||
|
||||
|
||||
EXECUTOR_VERSION = "dataflow-preview-v2"
|
||||
@@ -79,7 +90,10 @@ def execute_preview(
|
||||
row_limit: int,
|
||||
source_resolver: SourceResolver | None = None,
|
||||
preview_node_id: str | None = None,
|
||||
_execution_depth: int = 0,
|
||||
) -> PipelineExecutionResult:
|
||||
if _execution_depth > 5:
|
||||
raise PipelineExecutionError("Subflows are limited to five nested levels.")
|
||||
validation = validate_graph(graph)
|
||||
errors = [item for item in validation if item.severity == "error"]
|
||||
if errors:
|
||||
@@ -133,87 +147,37 @@ def execute_preview(
|
||||
node_preview=_node_preview(outputs, preview_node_id, row_limit),
|
||||
)
|
||||
try:
|
||||
if node.type == "source.inline":
|
||||
output_rows = [dict(row) for row in node.config.get("rows", [])]
|
||||
input_row_count += len(output_rows)
|
||||
source_fingerprints.append(
|
||||
{
|
||||
"node_id": node.id,
|
||||
"source_name": node.config.get("source_name"),
|
||||
"kind": "inline",
|
||||
"fingerprint": _rows_fingerprint(output_rows),
|
||||
"row_count": len(output_rows),
|
||||
}
|
||||
)
|
||||
elif node.type == "source.reference":
|
||||
if source_resolver is None:
|
||||
raise PipelineExecutionError(
|
||||
"Datasource-backed preview requires the Datasources catalogue capability.",
|
||||
node_id=node.id,
|
||||
)
|
||||
resolved = source_resolver(node, MAX_SOURCE_ROWS)
|
||||
output_rows = [dict(row) for row in resolved.rows]
|
||||
input_row_count += len(output_rows)
|
||||
source_fingerprints.append(
|
||||
{
|
||||
"node_id": node.id,
|
||||
"source_ref": resolved.source_ref,
|
||||
"source_name": node.config.get("source_name"),
|
||||
"kind": "datasource",
|
||||
"provider": resolved.provider,
|
||||
"fingerprint": resolved.fingerprint,
|
||||
"row_count": resolved.total_rows,
|
||||
"preview_rows": len(output_rows),
|
||||
"truncated": resolved.truncated,
|
||||
}
|
||||
)
|
||||
if resolved.truncated:
|
||||
message = (
|
||||
f"Source preview used {len(output_rows):,} of "
|
||||
f"{resolved.total_rows:,} rows."
|
||||
)
|
||||
node_messages.append(message)
|
||||
validation.append(
|
||||
DataflowDiagnostic(
|
||||
severity="warning",
|
||||
code="source.preview_truncated",
|
||||
message=message,
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
elif node.type == "combine.union":
|
||||
ordered_inputs = [
|
||||
outputs[source_id]
|
||||
for source_id in node_inputs.get("input", [])
|
||||
]
|
||||
output_rows = _union_rows(ordered_inputs, node.config)
|
||||
elif node.type == "combine.join":
|
||||
left_rows = outputs[node_inputs["left"][0]]
|
||||
right_rows = outputs[node_inputs["right"][0]]
|
||||
output_rows = _join_rows(
|
||||
left_rows,
|
||||
right_rows,
|
||||
node.config,
|
||||
executor = OPERATOR_REGISTRY.executor(node.type)
|
||||
if executor is None:
|
||||
raise PipelineExecutionError(
|
||||
f"Node type {node.type!r} has no registered executor.",
|
||||
node_id=node.id,
|
||||
)
|
||||
elif node.type == "filter":
|
||||
output_rows = _filter_rows(input_rows, node.config, node_id=node.id)
|
||||
elif node.type == "distinct":
|
||||
output_rows = _distinct_rows(input_rows, node.config)
|
||||
elif node.type == "select":
|
||||
output_rows = _select_rows(input_rows, node.config)
|
||||
elif node.type == "derive":
|
||||
output_rows = _derive_rows(input_rows, node.config, node_id=node.id)
|
||||
elif node.type == "aggregate":
|
||||
output_rows = _aggregate_rows(input_rows, node.config, node_id=node.id)
|
||||
elif node.type == "sort":
|
||||
output_rows = _sort_rows(input_rows, node.config)
|
||||
elif node.type == "limit":
|
||||
output_rows = input_rows[: int(node.config["count"])]
|
||||
elif node.type == "output":
|
||||
output_rows = [dict(row) for row in input_rows]
|
||||
else:
|
||||
raise PipelineExecutionError(f"Unsupported node type: {node.type}", node_id=node.id)
|
||||
execution = executor(
|
||||
OperatorExecutionContext(
|
||||
node=node,
|
||||
inputs_by_port=node_inputs,
|
||||
outputs=outputs,
|
||||
input_sets=input_sets,
|
||||
input_rows=input_rows,
|
||||
source_resolver=source_resolver,
|
||||
execution_depth=_execution_depth,
|
||||
)
|
||||
)
|
||||
output_rows = execution.rows
|
||||
input_row_count += execution.input_row_count
|
||||
source_fingerprints.extend(execution.source_fingerprints)
|
||||
node_messages.extend(execution.messages)
|
||||
if node.type == "source.reference" and execution.messages:
|
||||
validation.extend(
|
||||
DataflowDiagnostic(
|
||||
severity="warning",
|
||||
code="source.preview_truncated",
|
||||
message=message,
|
||||
node_id=node.id,
|
||||
)
|
||||
for message in execution.messages
|
||||
)
|
||||
if len(json.dumps(output_rows, default=str).encode("utf-8")) > MAX_RESULT_BYTES:
|
||||
raise PipelineExecutionError(
|
||||
"A preview node exceeded the one-megabyte result limit.",
|
||||
@@ -461,6 +425,309 @@ def _derive_value(operation: str, values: list[Any], *, separator: str) -> Any:
|
||||
raise ValueError(f"unknown derive operation {operation!r}")
|
||||
|
||||
|
||||
def _expression_filter_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
node_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
parsed = parse_expression(str(config["expression"]))
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
try:
|
||||
if bool(evaluate_expression(parsed, row)):
|
||||
result.append(dict(row))
|
||||
except (ArithmeticError, TypeError, ValueError) as exc:
|
||||
raise PipelineExecutionError(
|
||||
f"Cannot evaluate filter expression: {exc}",
|
||||
node_id=node_id,
|
||||
) from exc
|
||||
return result
|
||||
|
||||
|
||||
def _expression_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
node_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
target = str(config["target_column"])
|
||||
parsed = parse_expression(str(config["expression"]))
|
||||
output: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
try:
|
||||
item[target] = evaluate_expression(parsed, row)
|
||||
except (ArithmeticError, TypeError, ValueError) as exc:
|
||||
raise PipelineExecutionError(
|
||||
f"Cannot evaluate expression for {target!r}: {exc}",
|
||||
node_id=node_id,
|
||||
) from exc
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
|
||||
def _convert_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
node_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
source = str(config["source_column"])
|
||||
target = str(config["target_column"])
|
||||
target_type = str(config["target_type"])
|
||||
on_error = str(config.get("on_error", "fail"))
|
||||
output: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
try:
|
||||
item[target] = convert_value(
|
||||
row.get(source),
|
||||
target_type,
|
||||
on_error=on_error, # type: ignore[arg-type]
|
||||
)
|
||||
except (ArithmeticError, TypeError, ValueError) as exc:
|
||||
raise PipelineExecutionError(
|
||||
f"Cannot convert {source!r} to {target_type}: {exc}",
|
||||
node_id=node_id,
|
||||
) from exc
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
|
||||
def _replace_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
source = str(config["source_column"])
|
||||
target = str(config["target_column"])
|
||||
mode = str(config.get("mode", "exact"))
|
||||
find = config.get("find")
|
||||
replacement = config.get("replacement")
|
||||
output: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
value = row.get(source)
|
||||
if mode == "text" and value is not None:
|
||||
item[target] = str(value).replace(str(find), str(replacement))
|
||||
else:
|
||||
item[target] = replacement if value == find else value
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
|
||||
def _quality_rows(
|
||||
rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
node_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
rules = config.get("rules", [])
|
||||
action = str(config.get("action", "annotate"))
|
||||
unique_values: dict[str, set[object]] = defaultdict(set)
|
||||
output: list[dict[str, Any]] = []
|
||||
invalid_count = 0
|
||||
for row_index, row in enumerate(rows, start=1):
|
||||
failures: list[str] = []
|
||||
for rule in rules:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
rule_id = str(rule.get("id") or rule.get("operator") or "rule")
|
||||
column = str(rule.get("column") or "")
|
||||
value = row.get(column)
|
||||
operator = str(rule.get("operator") or "")
|
||||
valid = _quality_rule_matches(
|
||||
value,
|
||||
operator=operator,
|
||||
rule=rule,
|
||||
seen=unique_values[rule_id],
|
||||
)
|
||||
if not valid:
|
||||
failures.append(rule_id)
|
||||
if failures:
|
||||
invalid_count += 1
|
||||
if action == "fail":
|
||||
raise PipelineExecutionError(
|
||||
f"Quality rules failed for row {row_index}: {', '.join(failures)}.",
|
||||
node_id=node_id,
|
||||
)
|
||||
if action == "drop":
|
||||
continue
|
||||
item = dict(row)
|
||||
if action == "annotate":
|
||||
item["_quality_valid"] = not failures
|
||||
item["_quality_errors"] = failures
|
||||
output.append(item)
|
||||
if invalid_count and action not in {"annotate", "drop", "fail"}:
|
||||
raise PipelineExecutionError(
|
||||
f"Unsupported quality action {action!r}.",
|
||||
node_id=node_id,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _quality_rule_matches(
|
||||
value: Any,
|
||||
*,
|
||||
operator: str,
|
||||
rule: dict[str, Any],
|
||||
seen: set[object],
|
||||
) -> bool:
|
||||
if operator == "not_null":
|
||||
return value is not None and value != ""
|
||||
if operator == "type":
|
||||
expected = str(rule.get("value") or "")
|
||||
mapping = {
|
||||
"string": str,
|
||||
"integer": int,
|
||||
"number": (int, float, Decimal),
|
||||
"boolean": bool,
|
||||
}
|
||||
expected_type = mapping.get(expected)
|
||||
return expected_type is not None and isinstance(value, expected_type)
|
||||
if operator == "min":
|
||||
return value is not None and value >= rule.get("value")
|
||||
if operator == "max":
|
||||
return value is not None and value <= rule.get("value")
|
||||
if operator == "allowed":
|
||||
allowed = rule.get("values")
|
||||
return isinstance(allowed, list) and value in allowed
|
||||
if operator == "unique":
|
||||
marker = _hashable(value)
|
||||
if marker in seen:
|
||||
return False
|
||||
seen.add(marker)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _reconcile_rows(
|
||||
left_rows: list[dict[str, Any]],
|
||||
right_rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
node_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
left_keys = [str(item) for item in config["left_keys"]]
|
||||
right_keys = [str(item) for item in config["right_keys"]]
|
||||
right_prefix = str(config.get("right_prefix", "observed_"))
|
||||
comparisons = _comparison_fields(config.get("compare_columns"))
|
||||
left_index = _unique_row_index(left_rows, left_keys, node_id=node_id)
|
||||
right_index = _unique_row_index(right_rows, right_keys, node_id=node_id)
|
||||
output: list[dict[str, Any]] = []
|
||||
for key in dict.fromkeys((*left_index, *right_index)):
|
||||
left = left_index.get(key)
|
||||
right = right_index.get(key)
|
||||
item = dict(left or {})
|
||||
if right is not None:
|
||||
item.update({f"{right_prefix}{name}": value for name, value in right.items()})
|
||||
if left is None:
|
||||
status = "missing_expected"
|
||||
differences: list[str] = []
|
||||
elif right is None:
|
||||
status = "missing_observed"
|
||||
differences = []
|
||||
else:
|
||||
fields = comparisons or tuple((name, name) for name in left if name not in left_keys)
|
||||
differences = [
|
||||
left_name
|
||||
for left_name, right_name in fields
|
||||
if left.get(left_name) != right.get(right_name)
|
||||
]
|
||||
status = "changed" if differences else "match"
|
||||
item["_reconciliation_status"] = status
|
||||
item["_reconciliation_differences"] = differences
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
|
||||
def _comparison_fields(value: object) -> tuple[tuple[str, str], ...]:
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
fields: list[tuple[str, str]] = []
|
||||
for item in value:
|
||||
if isinstance(item, str) and item:
|
||||
fields.append((item, item))
|
||||
elif isinstance(item, dict):
|
||||
left = str(item.get("left") or item.get("column") or "")
|
||||
right = str(item.get("right") or left)
|
||||
if left and right:
|
||||
fields.append((left, right))
|
||||
return tuple(fields)
|
||||
|
||||
|
||||
def _unique_row_index(
|
||||
rows: list[dict[str, Any]],
|
||||
columns: list[str],
|
||||
*,
|
||||
node_id: str,
|
||||
) -> dict[tuple[Any, ...], dict[str, Any]]:
|
||||
index: dict[tuple[Any, ...], dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
key = tuple(_hashable(row.get(column)) for column in columns)
|
||||
if key in index:
|
||||
raise PipelineExecutionError(
|
||||
f"Reconciliation keys are not unique: {key!r}.",
|
||||
node_id=node_id,
|
||||
)
|
||||
index[key] = row
|
||||
return index
|
||||
|
||||
|
||||
def _execute_subflow(
|
||||
rows: list[dict[str, Any]],
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
source_resolver: SourceResolver | None,
|
||||
execution_depth: int,
|
||||
) -> PipelineExecutionResult:
|
||||
parameters = config.get("parameters")
|
||||
graph_payload = substitute_parameters(
|
||||
config.get("graph"),
|
||||
parameters if isinstance(parameters, dict) else {},
|
||||
)
|
||||
graph = PipelineGraph.model_validate(graph_payload)
|
||||
input_nodes = [
|
||||
node
|
||||
for node in graph.nodes
|
||||
if node.type == "source.inline" and node.config.get("input_binding") is True
|
||||
]
|
||||
if len(input_nodes) != 1:
|
||||
raise PipelineExecutionError(
|
||||
"A reusable subflow needs exactly one inline source with input_binding=true."
|
||||
)
|
||||
input_node = input_nodes[0]
|
||||
graph = graph.model_copy(
|
||||
update={
|
||||
"nodes": [
|
||||
(
|
||||
node.model_copy(
|
||||
update={
|
||||
"config": {
|
||||
**node.config,
|
||||
"rows": [dict(row) for row in rows],
|
||||
}
|
||||
},
|
||||
deep=True,
|
||||
)
|
||||
if node.id == input_node.id
|
||||
else node
|
||||
)
|
||||
for node in graph.nodes
|
||||
]
|
||||
},
|
||||
deep=True,
|
||||
)
|
||||
return execute_preview(
|
||||
graph,
|
||||
row_limit=MAX_INTERMEDIATE_ROWS,
|
||||
source_resolver=source_resolver,
|
||||
_execution_depth=execution_depth + 1,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
def _join_key(row: dict[str, Any], columns: list[str]) -> tuple[Any, ...] | None:
|
||||
values = tuple(row.get(column) for column in columns)
|
||||
if any(value is None for value in values):
|
||||
@@ -634,6 +901,12 @@ def _sortable_value(value: Any) -> tuple[str, Any]:
|
||||
return type(value).__name__, str(value)
|
||||
|
||||
|
||||
def _hashable(value: Any) -> Any:
|
||||
if isinstance(value, (dict, list, tuple, set)):
|
||||
return json.dumps(value, sort_keys=True, default=str)
|
||||
return value
|
||||
|
||||
|
||||
def _rows_fingerprint(rows: list[dict[str, Any]]) -> str:
|
||||
encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str)
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
@@ -657,6 +930,193 @@ def _type_name(value: Any) -> str:
|
||||
return type(value).__name__.lower()
|
||||
|
||||
|
||||
def _execute_inline_source(
|
||||
context: OperatorExecutionContext,
|
||||
) -> OperatorExecutionResult:
|
||||
rows = [dict(row) for row in context.node.config.get("rows", [])]
|
||||
return OperatorExecutionResult(
|
||||
rows=rows,
|
||||
input_row_count=len(rows),
|
||||
source_fingerprints=(
|
||||
{
|
||||
"node_id": context.node.id,
|
||||
"source_name": context.node.config.get("source_name"),
|
||||
"kind": "inline",
|
||||
"fingerprint": _rows_fingerprint(rows),
|
||||
"row_count": len(rows),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _execute_reference_source(
|
||||
context: OperatorExecutionContext,
|
||||
) -> OperatorExecutionResult:
|
||||
if context.source_resolver is None:
|
||||
raise PipelineExecutionError(
|
||||
"Datasource-backed preview requires the Datasources catalogue capability.",
|
||||
node_id=context.node.id,
|
||||
)
|
||||
resolved = context.source_resolver(context.node, MAX_SOURCE_ROWS)
|
||||
rows = [dict(row) for row in resolved.rows]
|
||||
messages = (
|
||||
(
|
||||
f"Source preview used {len(rows):,} of "
|
||||
f"{resolved.total_rows:,} rows."
|
||||
),
|
||||
) if resolved.truncated else ()
|
||||
return OperatorExecutionResult(
|
||||
rows=rows,
|
||||
messages=messages,
|
||||
input_row_count=len(rows),
|
||||
source_fingerprints=(
|
||||
{
|
||||
"node_id": context.node.id,
|
||||
"source_ref": resolved.source_ref,
|
||||
"source_name": context.node.config.get("source_name"),
|
||||
"kind": "datasource",
|
||||
"provider": resolved.provider,
|
||||
"fingerprint": resolved.fingerprint,
|
||||
"row_count": resolved.total_rows,
|
||||
"preview_rows": len(rows),
|
||||
"truncated": resolved.truncated,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _execute_subflow_node(
|
||||
context: OperatorExecutionContext,
|
||||
) -> OperatorExecutionResult:
|
||||
nested = _execute_subflow(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
source_resolver=context.source_resolver,
|
||||
execution_depth=context.execution_depth,
|
||||
)
|
||||
return OperatorExecutionResult(
|
||||
rows=nested.rows,
|
||||
messages=tuple(
|
||||
f"Subflow {item.node_id}: {message}"
|
||||
for item in nested.node_diagnostics
|
||||
for message in item.messages
|
||||
),
|
||||
source_fingerprints=tuple(
|
||||
{
|
||||
**fingerprint,
|
||||
"subflow_node_id": context.node.id,
|
||||
}
|
||||
for fingerprint in nested.source_fingerprints
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _register_executors() -> None:
|
||||
executors = {
|
||||
"source.inline": _execute_inline_source,
|
||||
"source.reference": _execute_reference_source,
|
||||
"combine.union": lambda context: OperatorExecutionResult(
|
||||
rows=_union_rows(
|
||||
[
|
||||
context.outputs[source_id]
|
||||
for source_id in context.inputs_by_port.get("input", ())
|
||||
],
|
||||
context.node.config,
|
||||
)
|
||||
),
|
||||
"combine.join": lambda context: OperatorExecutionResult(
|
||||
rows=_join_rows(
|
||||
context.outputs[context.inputs_by_port["left"][0]],
|
||||
context.outputs[context.inputs_by_port["right"][0]],
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"filter": lambda context: OperatorExecutionResult(
|
||||
rows=_filter_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"filter.expression": lambda context: OperatorExecutionResult(
|
||||
rows=_expression_filter_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"distinct": lambda context: OperatorExecutionResult(
|
||||
rows=_distinct_rows(context.input_rows, context.node.config)
|
||||
),
|
||||
"select": lambda context: OperatorExecutionResult(
|
||||
rows=_select_rows(context.input_rows, context.node.config)
|
||||
),
|
||||
"derive": lambda context: OperatorExecutionResult(
|
||||
rows=_derive_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"expression": lambda context: OperatorExecutionResult(
|
||||
rows=_expression_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"convert": lambda context: OperatorExecutionResult(
|
||||
rows=_convert_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"replace": lambda context: OperatorExecutionResult(
|
||||
rows=_replace_rows(context.input_rows, context.node.config)
|
||||
),
|
||||
"aggregate": lambda context: OperatorExecutionResult(
|
||||
rows=_aggregate_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"sort": lambda context: OperatorExecutionResult(
|
||||
rows=_sort_rows(context.input_rows, context.node.config)
|
||||
),
|
||||
"limit": lambda context: OperatorExecutionResult(
|
||||
rows=context.input_rows[: int(context.node.config["count"])]
|
||||
),
|
||||
"quality.rules": lambda context: OperatorExecutionResult(
|
||||
rows=_quality_rows(
|
||||
context.input_rows,
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"reconcile.compare": lambda context: OperatorExecutionResult(
|
||||
rows=_reconcile_rows(
|
||||
context.outputs[context.inputs_by_port["left"][0]],
|
||||
context.outputs[context.inputs_by_port["right"][0]],
|
||||
context.node.config,
|
||||
node_id=context.node.id,
|
||||
)
|
||||
),
|
||||
"subflow": _execute_subflow_node,
|
||||
"output": lambda context: OperatorExecutionResult(
|
||||
rows=[dict(row) for row in context.input_rows]
|
||||
),
|
||||
}
|
||||
for node_type, executor in executors.items():
|
||||
if OPERATOR_REGISTRY.executor(node_type) is None:
|
||||
OPERATOR_REGISTRY.register_executor(node_type, executor)
|
||||
|
||||
|
||||
_register_executors()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EXECUTOR_VERSION",
|
||||
"MAX_SOURCE_ROWS",
|
||||
|
||||
Reference in New Issue
Block a user