1644 lines
58 KiB
Python
1644 lines
58 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections import deque
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from govoplan_core.core.definition_graphs import (
|
|
DefinitionEdge,
|
|
DefinitionNode,
|
|
validate_definition_graph,
|
|
)
|
|
from govoplan_dataflow.backend.node_library import (
|
|
DATAFLOW_GRAPH_LIBRARY,
|
|
NODE_TYPES,
|
|
node_definition,
|
|
)
|
|
from govoplan_dataflow.backend.expressions import (
|
|
ExpressionError,
|
|
infer_expression_type,
|
|
parse_expression,
|
|
)
|
|
from govoplan_dataflow.backend.operator_registry import OPERATOR_REGISTRY
|
|
from govoplan_dataflow.backend.subflows import (
|
|
SubflowParameterError,
|
|
substitute_parameters,
|
|
)
|
|
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphEdge, GraphNode, PipelineGraph
|
|
|
|
|
|
SUPPORTED_NODE_TYPES = frozenset(NODE_TYPES)
|
|
FILTER_OPERATORS = frozenset(
|
|
{"eq", "ne", "gt", "gte", "lt", "lte", "contains", "is_null", "not_null"}
|
|
)
|
|
AGGREGATE_FUNCTIONS = frozenset({"count", "sum", "avg", "min", "max"})
|
|
DERIVE_OPERATIONS = frozenset(
|
|
{
|
|
"copy",
|
|
"upper",
|
|
"lower",
|
|
"trim",
|
|
"concat",
|
|
"coalesce",
|
|
"add",
|
|
"subtract",
|
|
"multiply",
|
|
"divide",
|
|
}
|
|
)
|
|
JOIN_TYPES = frozenset({"inner", "left", "right", "full"})
|
|
DATA_TYPES = frozenset(
|
|
{"string", "integer", "number", "boolean", "date", "datetime"}
|
|
)
|
|
QUALITY_OPERATORS = frozenset(
|
|
{"not_null", "type", "min", "max", "allowed", "unique"}
|
|
)
|
|
|
|
|
|
def canonical_graph_payload(graph: PipelineGraph) -> dict[str, Any]:
|
|
return graph.model_dump(mode="json", exclude_none=True)
|
|
|
|
|
|
def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str:
|
|
payload = {
|
|
"graph": canonical_graph_payload(graph),
|
|
"sql_text": (sql_text or "").strip() or None,
|
|
}
|
|
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def preserve_compatible_graph_layout(
|
|
reference: PipelineGraph,
|
|
compiled: PipelineGraph,
|
|
) -> PipelineGraph:
|
|
"""Keep graph identity and layout when SQL preserves the same topology."""
|
|
reference_keys = _structural_node_keys(reference)
|
|
compiled_keys = _structural_node_keys(compiled)
|
|
if reference_keys is None or compiled_keys is None:
|
|
return compiled
|
|
if (
|
|
len(set(reference_keys.values())) != len(reference_keys)
|
|
or len(set(compiled_keys.values())) != len(compiled_keys)
|
|
or set(reference_keys.values()) != set(compiled_keys.values())
|
|
):
|
|
return compiled
|
|
|
|
compiled_by_key = {
|
|
key: next(node for node in compiled.nodes if node.id == node_id)
|
|
for node_id, key in compiled_keys.items()
|
|
}
|
|
reference_edges = {
|
|
_structural_edge_key(edge, reference_keys): edge
|
|
for edge in reference.edges
|
|
}
|
|
compiled_edges = {
|
|
_structural_edge_key(edge, compiled_keys): edge
|
|
for edge in compiled.edges
|
|
}
|
|
if (
|
|
len(reference_edges) != len(reference.edges)
|
|
or len(compiled_edges) != len(compiled.edges)
|
|
or set(reference_edges) != set(compiled_edges)
|
|
):
|
|
return compiled
|
|
|
|
nodes = []
|
|
for reference_node in reference.nodes:
|
|
node = compiled_by_key[reference_keys[reference_node.id]]
|
|
nodes.append(
|
|
node.model_copy(
|
|
update={
|
|
"id": reference_node.id,
|
|
"label": reference_node.label,
|
|
"position": reference_node.position.model_copy(deep=True),
|
|
},
|
|
deep=True,
|
|
)
|
|
)
|
|
edges = []
|
|
for reference_edge in reference.edges:
|
|
edge_key = _structural_edge_key(reference_edge, reference_keys)
|
|
edge = compiled_edges[edge_key]
|
|
edges.append(
|
|
edge.model_copy(
|
|
update={
|
|
"id": reference_edge.id,
|
|
"source": reference_edge.source,
|
|
"target": reference_edge.target,
|
|
},
|
|
deep=True,
|
|
)
|
|
)
|
|
return compiled.model_copy(
|
|
update={"nodes": nodes, "edges": edges},
|
|
deep=True,
|
|
)
|
|
|
|
|
|
def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
|
diagnostics = [
|
|
DataflowDiagnostic(
|
|
severity=item.severity,
|
|
code=item.code,
|
|
message=item.message,
|
|
node_id=item.node_id,
|
|
field=item.field,
|
|
)
|
|
for item in validate_definition_graph(
|
|
DATAFLOW_GRAPH_LIBRARY,
|
|
nodes=tuple(DefinitionNode(id=node.id, type=node.type) for node in graph.nodes),
|
|
edges=tuple(
|
|
DefinitionEdge(
|
|
id=edge.id,
|
|
source=edge.source,
|
|
target=edge.target,
|
|
source_port=edge.source_port,
|
|
target_port=edge.target_port,
|
|
)
|
|
for edge in graph.edges
|
|
),
|
|
)
|
|
]
|
|
nodes = {node.id: node for node in graph.nodes}
|
|
incoming: dict[str, list[GraphEdge]] = {node_id: [] for node_id in nodes}
|
|
outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
|
for edge in graph.edges:
|
|
if edge.source not in nodes or edge.target not in nodes or edge.source == edge.target:
|
|
continue
|
|
source_definition = node_definition(nodes[edge.source].type)
|
|
target_definition = node_definition(nodes[edge.target].type)
|
|
if (
|
|
source_definition is None
|
|
or target_definition is None
|
|
or edge.source_port not in {port.id for port in source_definition.output_ports}
|
|
or edge.target_port not in {port.id for port in target_definition.input_ports}
|
|
):
|
|
continue
|
|
outgoing[edge.source].append(edge.target)
|
|
incoming[edge.target].append(edge)
|
|
|
|
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
|
|
output_nodes = [node for node in graph.nodes if node.type == "output"]
|
|
source_names = [
|
|
str(node.config.get("source_name", "")).strip()
|
|
for node in source_nodes
|
|
if _non_empty_text(node.config.get("source_name"))
|
|
]
|
|
duplicate_source_names = sorted(
|
|
{
|
|
name
|
|
for name in source_names
|
|
if sum(candidate.casefold() == name.casefold() for candidate in source_names) > 1
|
|
},
|
|
key=str.casefold,
|
|
)
|
|
if duplicate_source_names:
|
|
diagnostics.append(
|
|
_error(
|
|
"source.duplicate_name",
|
|
"Logical source names must be unique: "
|
|
f"{', '.join(duplicate_source_names)}.",
|
|
)
|
|
)
|
|
for node in graph.nodes:
|
|
if node.type not in SUPPORTED_NODE_TYPES:
|
|
continue
|
|
validator = OPERATOR_REGISTRY.config_validator(node.type)
|
|
if validator is None:
|
|
diagnostics.append(
|
|
_error(
|
|
"node.validator_missing",
|
|
f"Node type {node.type!r} has no registered validator.",
|
|
node_id=node.id,
|
|
)
|
|
)
|
|
continue
|
|
diagnostics.extend(validator(node))
|
|
|
|
ordered, cyclic = topological_order(graph)
|
|
if not cyclic and source_nodes and len(output_nodes) == 1:
|
|
reachable: set[str] = set()
|
|
for source in source_nodes:
|
|
reachable.update(_reachable_from(source.id, outgoing))
|
|
if len(reachable) != len(nodes):
|
|
diagnostics.append(
|
|
_error("graph.disconnected", "Every node must be connected to a pipeline source.")
|
|
)
|
|
reverse_adjacency = {
|
|
node_id: [edge.source for edge in edges]
|
|
for node_id, edges in incoming.items()
|
|
}
|
|
reaches_output = _reachable_from(output_nodes[0].id, reverse_adjacency)
|
|
if len(reaches_output) != len(nodes):
|
|
diagnostics.append(
|
|
_error("graph.dead_end", "Every node must lead to the pipeline output.")
|
|
)
|
|
if ordered and ordered[-1] != output_nodes[0].id:
|
|
diagnostics.append(
|
|
_error("graph.output_not_terminal", "The output node must be the terminal transform.")
|
|
)
|
|
diagnostics.extend(_validate_graph_schemas(graph, ordered=ordered))
|
|
return _dedupe_diagnostics(diagnostics)
|
|
|
|
|
|
def _structural_node_keys(
|
|
graph: PipelineGraph,
|
|
) -> dict[str, tuple[object, ...]] | None:
|
|
node_by_id = {node.id: node for node in graph.nodes}
|
|
if (
|
|
len(node_by_id) != len(graph.nodes)
|
|
or any(
|
|
edge.source not in node_by_id or edge.target not in node_by_id
|
|
for edge in graph.edges
|
|
)
|
|
):
|
|
return None
|
|
ordered, cyclic = topological_order(graph)
|
|
if cyclic:
|
|
return None
|
|
incoming: dict[str, list[GraphEdge]] = {
|
|
node.id: []
|
|
for node in graph.nodes
|
|
}
|
|
for edge in graph.edges:
|
|
if edge.target in incoming:
|
|
incoming[edge.target].append(edge)
|
|
keys: dict[str, tuple[object, ...]] = {}
|
|
for node_id in ordered:
|
|
node = node_by_id[node_id]
|
|
input_keys: list[tuple[object, ...]] = []
|
|
for edge in incoming[node_id]:
|
|
source_key = keys.get(edge.source)
|
|
if source_key is None:
|
|
return None
|
|
input_keys.append(
|
|
(
|
|
edge.target_port,
|
|
edge.source_port,
|
|
source_key,
|
|
)
|
|
)
|
|
if node.type != "combine.union":
|
|
input_keys.sort(key=repr)
|
|
source_name = (
|
|
str(node.config.get("source_name", "")).strip().casefold()
|
|
if node.type.startswith("source.")
|
|
else None
|
|
)
|
|
keys[node_id] = (
|
|
node.type,
|
|
source_name,
|
|
tuple(input_keys),
|
|
)
|
|
return keys
|
|
|
|
|
|
def _structural_edge_key(
|
|
edge: GraphEdge,
|
|
node_keys: dict[str, tuple[object, ...]],
|
|
) -> tuple[object, ...]:
|
|
return (
|
|
node_keys[edge.source],
|
|
node_keys[edge.target],
|
|
edge.source_port,
|
|
edge.target_port,
|
|
)
|
|
|
|
|
|
def _dedupe_diagnostics(
|
|
diagnostics: list[DataflowDiagnostic],
|
|
) -> list[DataflowDiagnostic]:
|
|
seen: set[tuple[str, str | None, str | None]] = set()
|
|
result: list[DataflowDiagnostic] = []
|
|
for item in diagnostics:
|
|
key = (item.code, item.node_id, item.field)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
result.append(item)
|
|
return result
|
|
|
|
|
|
def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]:
|
|
node_ids = [node.id for node in graph.nodes]
|
|
incoming_count = {node_id: 0 for node_id in node_ids}
|
|
outgoing: dict[str, list[str]] = {node_id: [] for node_id in node_ids}
|
|
for edge in graph.edges:
|
|
if edge.source in outgoing and edge.target in incoming_count and edge.source != edge.target:
|
|
outgoing[edge.source].append(edge.target)
|
|
incoming_count[edge.target] += 1
|
|
|
|
ready = deque(node_id for node_id in node_ids if incoming_count[node_id] == 0)
|
|
ordered: list[str] = []
|
|
while ready:
|
|
node_id = ready.popleft()
|
|
ordered.append(node_id)
|
|
for target in outgoing[node_id]:
|
|
incoming_count[target] -= 1
|
|
if incoming_count[target] == 0:
|
|
ready.append(target)
|
|
return ordered, len(ordered) != len(node_ids)
|
|
|
|
|
|
def graph_inputs_by_port(graph: PipelineGraph) -> dict[str, dict[str, list[str]]]:
|
|
result: dict[str, dict[str, list[str]]] = {}
|
|
for edge in graph.edges:
|
|
result.setdefault(edge.target, {}).setdefault(edge.target_port, []).append(edge.source)
|
|
return result
|
|
|
|
|
|
def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]:
|
|
seen: set[str] = set()
|
|
pending = [start]
|
|
while pending:
|
|
node_id = pending.pop()
|
|
if node_id in seen:
|
|
continue
|
|
seen.add(node_id)
|
|
pending.extend(adjacency.get(node_id, ()))
|
|
return seen
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SchemaState:
|
|
columns: frozenset[str]
|
|
open: bool = False
|
|
types: dict[str, str] = field(default_factory=dict)
|
|
|
|
def knows(self, column: str) -> bool:
|
|
return self.open or column in self.columns
|
|
|
|
def type_of(self, column: str) -> str:
|
|
return self.types.get(column, "unknown")
|
|
|
|
|
|
def _validate_graph_schemas(
|
|
graph: PipelineGraph,
|
|
*,
|
|
ordered: list[str],
|
|
) -> list[DataflowDiagnostic]:
|
|
diagnostics: list[DataflowDiagnostic] = []
|
|
node_by_id = {node.id: node for node in graph.nodes}
|
|
inputs = graph_inputs_by_port(graph)
|
|
schemas: dict[str, _SchemaState] = {}
|
|
for node_id in ordered:
|
|
node = node_by_id[node_id]
|
|
node_inputs = inputs.get(node.id, {})
|
|
input_states = [
|
|
schemas[source_id]
|
|
for port_sources in node_inputs.values()
|
|
for source_id in port_sources
|
|
if source_id in schemas
|
|
]
|
|
input_state = input_states[0] if input_states else _SchemaState(frozenset(), open=True)
|
|
if node.type == "source.inline":
|
|
rows = node.config.get("rows")
|
|
schemas[node.id] = _inline_schema(rows)
|
|
continue
|
|
if node.type == "source.reference":
|
|
schemas[node.id] = _configured_source_schema(
|
|
node.config.get("source_columns")
|
|
)
|
|
continue
|
|
if node.type == "combine.union":
|
|
if len(input_states) > 1:
|
|
closed_shapes = {
|
|
state.columns
|
|
for state in input_states
|
|
if not state.open
|
|
}
|
|
if len(closed_shapes) > 1:
|
|
diagnostics.append(
|
|
_warning(
|
|
"union.schema_mismatch",
|
|
"Appended inputs use different columns; missing values will be null.",
|
|
node_id=node.id,
|
|
)
|
|
)
|
|
schemas[node.id] = _SchemaState(
|
|
frozenset().union(*(state.columns for state in input_states)),
|
|
open=any(state.open for state in input_states),
|
|
types=_merged_schema_types(input_states),
|
|
)
|
|
continue
|
|
if node.type == "combine.join":
|
|
left_state = _port_schema(node_inputs, schemas, "left")
|
|
right_state = _port_schema(node_inputs, schemas, "right")
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=left_state,
|
|
columns=node.config.get("left_keys"),
|
|
field="left_keys",
|
|
)
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=right_state,
|
|
columns=node.config.get("right_keys"),
|
|
field="right_keys",
|
|
)
|
|
prefix = str(node.config.get("right_prefix", "right_"))
|
|
prefixed_right = {f"{prefix}{column}" for column in right_state.columns}
|
|
collisions = left_state.columns & prefixed_right
|
|
if collisions:
|
|
diagnostics.append(
|
|
_error(
|
|
"join.output_collision",
|
|
f"Join output columns collide: {', '.join(sorted(collisions))}.",
|
|
node_id=node.id,
|
|
field="right_prefix",
|
|
)
|
|
)
|
|
schemas[node.id] = _SchemaState(
|
|
frozenset(left_state.columns | prefixed_right),
|
|
open=left_state.open or right_state.open,
|
|
types={
|
|
**left_state.types,
|
|
**{
|
|
f"{prefix}{column}": right_state.type_of(column)
|
|
for column in right_state.columns
|
|
},
|
|
},
|
|
)
|
|
continue
|
|
if node.type == "filter":
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=[node.config.get("column")],
|
|
field="column",
|
|
)
|
|
schemas[node.id] = input_state
|
|
continue
|
|
if node.type == "filter.expression":
|
|
parsed = _parsed_node_expression(
|
|
diagnostics,
|
|
node=node,
|
|
field_name="expression",
|
|
)
|
|
if parsed is not None:
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=list(parsed.columns),
|
|
field="expression",
|
|
)
|
|
schemas[node.id] = input_state
|
|
continue
|
|
if node.type == "distinct":
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=node.config.get("columns"),
|
|
field="columns",
|
|
)
|
|
schemas[node.id] = input_state
|
|
continue
|
|
if node.type == "select":
|
|
fields = node.config.get("fields")
|
|
selected_columns: list[str] = []
|
|
output_columns: list[str] = []
|
|
if isinstance(fields, list):
|
|
for field in fields:
|
|
if isinstance(field, str):
|
|
selected_columns.append(field)
|
|
output_columns.append(field)
|
|
elif isinstance(field, dict):
|
|
column = field.get("column")
|
|
alias = field.get("alias") or column
|
|
if isinstance(column, str):
|
|
selected_columns.append(column)
|
|
if isinstance(alias, str):
|
|
output_columns.append(alias)
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=selected_columns,
|
|
field="fields",
|
|
)
|
|
_validate_output_names(
|
|
diagnostics,
|
|
node=node,
|
|
columns=output_columns,
|
|
field="fields",
|
|
)
|
|
schemas[node.id] = _SchemaState(
|
|
frozenset(output_columns),
|
|
types={
|
|
output: input_state.type_of(source)
|
|
for source, output in zip(
|
|
selected_columns,
|
|
output_columns,
|
|
strict=False,
|
|
)
|
|
},
|
|
)
|
|
continue
|
|
if node.type == "derive":
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=node.config.get("source_columns"),
|
|
field="source_columns",
|
|
)
|
|
target = node.config.get("target_column")
|
|
if isinstance(target, str) and target:
|
|
if target in input_state.columns:
|
|
diagnostics.append(
|
|
_warning(
|
|
"derive.overwrites_column",
|
|
f"Derived column {target!r} replaces an existing value.",
|
|
node_id=node.id,
|
|
field="target_column",
|
|
)
|
|
)
|
|
schemas[node.id] = _SchemaState(
|
|
input_state.columns | frozenset((target,)),
|
|
open=input_state.open,
|
|
types={
|
|
**input_state.types,
|
|
target: _derive_result_type(
|
|
str(node.config.get("operation") or ""),
|
|
[
|
|
input_state.type_of(column)
|
|
for column in _text_items(
|
|
node.config.get("source_columns")
|
|
)
|
|
],
|
|
),
|
|
},
|
|
)
|
|
else:
|
|
schemas[node.id] = input_state
|
|
continue
|
|
if node.type == "expression":
|
|
parsed = _parsed_node_expression(
|
|
diagnostics,
|
|
node=node,
|
|
field_name="expression",
|
|
)
|
|
target = str(node.config.get("target_column") or "")
|
|
result_type = "unknown"
|
|
if parsed is not None:
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=list(parsed.columns),
|
|
field="expression",
|
|
)
|
|
result_type = infer_expression_type(
|
|
parsed,
|
|
{
|
|
name: input_state.type_of(name) # type: ignore[dict-item]
|
|
for name in input_state.columns
|
|
},
|
|
)
|
|
expected = str(node.config.get("result_type") or "unknown")
|
|
if (
|
|
expected != "unknown"
|
|
and result_type not in {"unknown", "null", expected}
|
|
):
|
|
diagnostics.append(
|
|
_warning(
|
|
"expression.type_mismatch",
|
|
f"Expression infers {result_type}, not {expected}.",
|
|
node_id=node.id,
|
|
field="result_type",
|
|
)
|
|
)
|
|
schemas[node.id] = _SchemaState(
|
|
input_state.columns | ({target} if target else set()),
|
|
open=input_state.open,
|
|
types={
|
|
**input_state.types,
|
|
**(
|
|
{target: expected if expected != "unknown" else result_type}
|
|
if target
|
|
else {}
|
|
),
|
|
},
|
|
)
|
|
continue
|
|
if node.type in {"convert", "replace"}:
|
|
source = str(node.config.get("source_column") or "")
|
|
target = str(node.config.get("target_column") or "")
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=[source],
|
|
field="source_column",
|
|
)
|
|
target_type = (
|
|
str(node.config.get("target_type") or "unknown")
|
|
if node.type == "convert"
|
|
else input_state.type_of(source)
|
|
)
|
|
schemas[node.id] = _SchemaState(
|
|
input_state.columns | ({target} if target else set()),
|
|
open=input_state.open,
|
|
types={
|
|
**input_state.types,
|
|
**({target: target_type} if target else {}),
|
|
},
|
|
)
|
|
continue
|
|
if node.type == "aggregate":
|
|
group_by = _text_items(node.config.get("group_by"))
|
|
aggregates = node.config.get("aggregates")
|
|
aggregate_columns = [
|
|
str(item.get("column"))
|
|
for item in aggregates
|
|
if isinstance(aggregates, list)
|
|
and isinstance(item, dict)
|
|
and item.get("column") not in (None, "", "*")
|
|
] if isinstance(aggregates, list) else []
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=[*group_by, *aggregate_columns],
|
|
field="aggregates",
|
|
)
|
|
aliases = [
|
|
str(item.get("alias"))
|
|
for item in aggregates
|
|
if isinstance(aggregates, list)
|
|
and isinstance(item, dict)
|
|
and item.get("alias")
|
|
] if isinstance(aggregates, list) else []
|
|
output_columns = [*group_by, *aliases]
|
|
_validate_output_names(
|
|
diagnostics,
|
|
node=node,
|
|
columns=output_columns,
|
|
field="aggregates",
|
|
)
|
|
aggregate_types = {
|
|
str(item.get("alias")): (
|
|
"integer"
|
|
if item.get("function") == "count"
|
|
else input_state.type_of(str(item.get("column") or ""))
|
|
)
|
|
for item in aggregates
|
|
if isinstance(aggregates, list)
|
|
and isinstance(item, dict)
|
|
and item.get("alias")
|
|
} if isinstance(aggregates, list) else {}
|
|
schemas[node.id] = _SchemaState(
|
|
frozenset(output_columns),
|
|
types={
|
|
**{
|
|
column: input_state.type_of(column)
|
|
for column in group_by
|
|
},
|
|
**aggregate_types,
|
|
},
|
|
)
|
|
continue
|
|
if node.type == "sort":
|
|
fields = node.config.get("fields")
|
|
columns = [
|
|
str(item.get("column"))
|
|
for item in fields
|
|
if isinstance(fields, list)
|
|
and isinstance(item, dict)
|
|
and item.get("column")
|
|
] if isinstance(fields, list) else []
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=columns,
|
|
field="fields",
|
|
)
|
|
schemas[node.id] = input_state
|
|
continue
|
|
if node.type == "quality.rules":
|
|
rules = node.config.get("rules")
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=input_state,
|
|
columns=[
|
|
str(rule.get("column") or "")
|
|
for rule in rules
|
|
if isinstance(rules, list) and isinstance(rule, dict)
|
|
] if isinstance(rules, list) else [],
|
|
field="rules",
|
|
)
|
|
if node.config.get("action", "annotate") == "annotate":
|
|
schemas[node.id] = _SchemaState(
|
|
input_state.columns
|
|
| frozenset(("_quality_valid", "_quality_errors")),
|
|
open=input_state.open,
|
|
types={
|
|
**input_state.types,
|
|
"_quality_valid": "boolean",
|
|
"_quality_errors": "array",
|
|
},
|
|
)
|
|
else:
|
|
schemas[node.id] = input_state
|
|
continue
|
|
if node.type == "reconcile.compare":
|
|
left_state = _port_schema(node_inputs, schemas, "left")
|
|
right_state = _port_schema(node_inputs, schemas, "right")
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=left_state,
|
|
columns=node.config.get("left_keys"),
|
|
field="left_keys",
|
|
)
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=right_state,
|
|
columns=node.config.get("right_keys"),
|
|
field="right_keys",
|
|
)
|
|
comparison_columns = node.config.get("compare_columns")
|
|
if isinstance(comparison_columns, list):
|
|
left_columns: list[str] = []
|
|
right_columns: list[str] = []
|
|
for item in comparison_columns:
|
|
if isinstance(item, str):
|
|
left_columns.append(item)
|
|
right_columns.append(item)
|
|
elif isinstance(item, dict):
|
|
left_columns.append(
|
|
str(item.get("left") or item.get("column") or "")
|
|
)
|
|
right_columns.append(
|
|
str(
|
|
item.get("right")
|
|
or item.get("left")
|
|
or item.get("column")
|
|
or ""
|
|
)
|
|
)
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=left_state,
|
|
columns=left_columns,
|
|
field="compare_columns",
|
|
)
|
|
_validate_columns(
|
|
diagnostics,
|
|
node=node,
|
|
state=right_state,
|
|
columns=right_columns,
|
|
field="compare_columns",
|
|
)
|
|
prefix = str(node.config.get("right_prefix") or "observed_")
|
|
schemas[node.id] = _SchemaState(
|
|
left_state.columns
|
|
| frozenset(f"{prefix}{column}" for column in right_state.columns)
|
|
| frozenset(
|
|
(
|
|
"_reconciliation_status",
|
|
"_reconciliation_differences",
|
|
)
|
|
),
|
|
open=left_state.open or right_state.open,
|
|
types={
|
|
**left_state.types,
|
|
**{
|
|
f"{prefix}{column}": right_state.type_of(column)
|
|
for column in right_state.columns
|
|
},
|
|
"_reconciliation_status": "string",
|
|
"_reconciliation_differences": "array",
|
|
},
|
|
)
|
|
continue
|
|
if node.type == "subflow":
|
|
output_schema = _configured_source_schema(
|
|
node.config.get("output_schema")
|
|
)
|
|
schemas[node.id] = (
|
|
output_schema
|
|
if output_schema.columns
|
|
else _SchemaState(frozenset(), open=True)
|
|
)
|
|
continue
|
|
schemas[node.id] = input_state
|
|
return diagnostics
|
|
|
|
|
|
def _configured_source_schema(value: object) -> _SchemaState:
|
|
if not isinstance(value, list):
|
|
return _SchemaState(frozenset(), open=True)
|
|
columns = {
|
|
item if isinstance(item, str) else str(item.get("name"))
|
|
for item in value
|
|
if (
|
|
isinstance(item, str) and item
|
|
or isinstance(item, dict) and item.get("name")
|
|
)
|
|
}
|
|
types = {
|
|
str(item.get("name")): str(
|
|
item.get("data_type") or item.get("type") or "unknown"
|
|
)
|
|
for item in value
|
|
if isinstance(item, dict) and item.get("name")
|
|
}
|
|
return _SchemaState(
|
|
frozenset(columns),
|
|
open=not columns,
|
|
types=types,
|
|
)
|
|
|
|
|
|
def _inline_schema(value: object) -> _SchemaState:
|
|
if not isinstance(value, list):
|
|
return _SchemaState(frozenset(), open=True)
|
|
columns = {
|
|
str(column)
|
|
for row in value
|
|
if isinstance(row, dict)
|
|
for column in row
|
|
}
|
|
types: dict[str, str] = {}
|
|
for column in columns:
|
|
observed = {
|
|
_schema_value_type(row.get(column))
|
|
for row in value
|
|
if isinstance(row, dict) and row.get(column) is not None
|
|
}
|
|
types[column] = observed.pop() if len(observed) == 1 else "unknown"
|
|
return _SchemaState(
|
|
frozenset(columns),
|
|
open=not columns,
|
|
types=types,
|
|
)
|
|
|
|
|
|
def _schema_value_type(value: object) -> str:
|
|
if isinstance(value, bool):
|
|
return "boolean"
|
|
if isinstance(value, int):
|
|
return "integer"
|
|
if isinstance(value, float):
|
|
return "number"
|
|
if isinstance(value, str):
|
|
return "string"
|
|
if isinstance(value, list):
|
|
return "array"
|
|
if isinstance(value, dict):
|
|
return "object"
|
|
return "unknown"
|
|
|
|
|
|
def _merged_schema_types(states: list[_SchemaState]) -> dict[str, str]:
|
|
columns = frozenset().union(*(state.columns for state in states))
|
|
types: dict[str, str] = {}
|
|
for column in columns:
|
|
observed = {
|
|
state.type_of(column)
|
|
for state in states
|
|
if column in state.columns and state.type_of(column) != "unknown"
|
|
}
|
|
types[column] = observed.pop() if len(observed) == 1 else "unknown"
|
|
return types
|
|
|
|
|
|
def _derive_result_type(operation: str, source_types: list[str]) -> str:
|
|
if operation in {"upper", "lower", "trim", "concat"}:
|
|
return "string"
|
|
if operation in {"add", "subtract", "multiply", "divide"}:
|
|
return "number" if "number" in source_types or operation == "divide" else "integer"
|
|
if operation in {"copy", "coalesce"}:
|
|
concrete = {
|
|
item for item in source_types if item not in {"unknown", "null"}
|
|
}
|
|
return concrete.pop() if len(concrete) == 1 else "unknown"
|
|
return "unknown"
|
|
|
|
|
|
def _parsed_node_expression(
|
|
diagnostics: list[DataflowDiagnostic],
|
|
*,
|
|
node: GraphNode,
|
|
field_name: str,
|
|
):
|
|
try:
|
|
return parse_expression(str(node.config.get(field_name) or ""))
|
|
except ExpressionError as exc:
|
|
diagnostics.append(
|
|
_error(
|
|
"expression.invalid",
|
|
str(exc),
|
|
node_id=node.id,
|
|
field=field_name,
|
|
)
|
|
)
|
|
return None
|
|
|
|
|
|
def _port_schema(
|
|
inputs: dict[str, list[str]],
|
|
schemas: dict[str, _SchemaState],
|
|
port: str,
|
|
) -> _SchemaState:
|
|
source_ids = inputs.get(port, [])
|
|
return schemas.get(source_ids[0], _SchemaState(frozenset(), open=True)) if source_ids else _SchemaState(frozenset(), open=True)
|
|
|
|
|
|
def _validate_columns(
|
|
diagnostics: list[DataflowDiagnostic],
|
|
*,
|
|
node: GraphNode,
|
|
state: _SchemaState,
|
|
columns: object,
|
|
field: str,
|
|
) -> None:
|
|
for column in _text_items(columns):
|
|
if not state.knows(column):
|
|
diagnostics.append(
|
|
_error(
|
|
"schema.unknown_column",
|
|
f"Column {column!r} is not available at this node.",
|
|
node_id=node.id,
|
|
field=field,
|
|
)
|
|
)
|
|
|
|
|
|
def _validate_output_names(
|
|
diagnostics: list[DataflowDiagnostic],
|
|
*,
|
|
node: GraphNode,
|
|
columns: list[str],
|
|
field: str,
|
|
) -> None:
|
|
duplicates = sorted({column for column in columns if columns.count(column) > 1})
|
|
if duplicates:
|
|
diagnostics.append(
|
|
_error(
|
|
"schema.duplicate_output",
|
|
f"Output column names must be unique: {', '.join(duplicates)}.",
|
|
node_id=node.id,
|
|
field=field,
|
|
)
|
|
)
|
|
|
|
|
|
def _text_items(value: object) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [item for item in value if isinstance(item, str) and item]
|
|
|
|
|
|
def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
|
config = node.config
|
|
diagnostics: list[DataflowDiagnostic] = []
|
|
if node.type in {"source.inline", "source.reference"}:
|
|
source_name = config.get("source_name")
|
|
if not isinstance(source_name, str) or not source_name.strip():
|
|
diagnostics.append(
|
|
_error(
|
|
"source.name_required",
|
|
"A source needs a logical SQL name.",
|
|
node_id=node.id,
|
|
field="source_name",
|
|
)
|
|
)
|
|
elif re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", source_name.strip()) is None:
|
|
diagnostics.append(
|
|
_error(
|
|
"source.name_invalid",
|
|
"Logical source names must be SQL identifiers, such as monthly_cases.",
|
|
node_id=node.id,
|
|
field="source_name",
|
|
)
|
|
)
|
|
if node.type == "source.reference":
|
|
if not _non_empty_text(config.get("source_ref")):
|
|
diagnostics.append(
|
|
_error(
|
|
"source.reference_required",
|
|
"Choose a datasource.",
|
|
node_id=node.id,
|
|
field="source_ref",
|
|
)
|
|
)
|
|
expected_fingerprint = config.get("expected_fingerprint")
|
|
if expected_fingerprint is not None and not isinstance(expected_fingerprint, str):
|
|
diagnostics.append(
|
|
_error(
|
|
"source.fingerprint",
|
|
"The expected source fingerprint must be text.",
|
|
node_id=node.id,
|
|
field="expected_fingerprint",
|
|
)
|
|
)
|
|
return diagnostics
|
|
rows = config.get("rows")
|
|
if not isinstance(rows, list):
|
|
diagnostics.append(
|
|
_error("source.rows_required", "Inline source rows must be a list.", node_id=node.id, field="rows")
|
|
)
|
|
elif len(rows) > 250:
|
|
diagnostics.append(
|
|
_error("source.row_limit", "Inline sources are limited to 250 rows.", node_id=node.id, field="rows")
|
|
)
|
|
elif any(not isinstance(row, dict) for row in rows):
|
|
diagnostics.append(
|
|
_error("source.row_shape", "Every inline source row must be an object.", node_id=node.id, field="rows")
|
|
)
|
|
if len(json.dumps(config, default=str).encode("utf-8")) > 512_000:
|
|
diagnostics.append(
|
|
_error("source.byte_limit", "Inline source configuration is limited to 512 KiB.", node_id=node.id)
|
|
)
|
|
elif node.type == "filter":
|
|
if not _non_empty_text(config.get("column")):
|
|
diagnostics.append(_node_field_error(node, "filter.column", "Choose a column.", "column"))
|
|
operator = config.get("operator")
|
|
if operator not in FILTER_OPERATORS:
|
|
diagnostics.append(
|
|
_node_field_error(node, "filter.operator", "Choose a supported comparison operator.", "operator")
|
|
)
|
|
if operator not in {"is_null", "not_null"} and "value" not in config:
|
|
diagnostics.append(_node_field_error(node, "filter.value", "Enter a comparison value.", "value"))
|
|
elif node.type == "filter.expression":
|
|
if not _non_empty_text(config.get("expression")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"expression.required",
|
|
"Enter a filter expression.",
|
|
"expression",
|
|
)
|
|
)
|
|
elif node.type == "distinct":
|
|
columns = config.get("columns", [])
|
|
if not isinstance(columns, list) or any(not _non_empty_text(item) for item in columns):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"distinct.columns",
|
|
"Distinct key columns must be named.",
|
|
"columns",
|
|
)
|
|
)
|
|
elif node.type == "combine.union":
|
|
if config.get("mode", "all") not in {"all", "distinct"}:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"union.mode",
|
|
"Choose whether duplicate rows are kept or removed.",
|
|
"mode",
|
|
)
|
|
)
|
|
elif node.type == "combine.join":
|
|
if config.get("join_type", "inner") not in JOIN_TYPES:
|
|
diagnostics.append(
|
|
_node_field_error(node, "join.type", "Choose a supported join type.", "join_type")
|
|
)
|
|
left_keys = config.get("left_keys")
|
|
right_keys = config.get("right_keys")
|
|
if (
|
|
not isinstance(left_keys, list)
|
|
or not left_keys
|
|
or any(not _non_empty_text(item) for item in left_keys)
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(node, "join.left_keys", "Add at least one left key.", "left_keys")
|
|
)
|
|
if (
|
|
not isinstance(right_keys, list)
|
|
or not right_keys
|
|
or any(not _non_empty_text(item) for item in right_keys)
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(node, "join.right_keys", "Add at least one right key.", "right_keys")
|
|
)
|
|
if isinstance(left_keys, list) and isinstance(right_keys, list) and len(left_keys) != len(right_keys):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"join.key_count",
|
|
"Left and right joins need the same number of keys.",
|
|
"right_keys",
|
|
)
|
|
)
|
|
right_prefix = config.get("right_prefix", "right_")
|
|
if (
|
|
not _non_empty_text(right_prefix)
|
|
or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*_", str(right_prefix)) is None
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"join.right_prefix",
|
|
"Enter an identifier prefix ending in an underscore, such as right_.",
|
|
"right_prefix",
|
|
)
|
|
)
|
|
elif node.type == "select":
|
|
fields = config.get("fields")
|
|
if not isinstance(fields, list) or not fields:
|
|
diagnostics.append(
|
|
_node_field_error(node, "select.fields", "Select at least one output column.", "fields")
|
|
)
|
|
else:
|
|
for field in fields:
|
|
column = field if isinstance(field, str) else field.get("column") if isinstance(field, dict) else None
|
|
if not _non_empty_text(column):
|
|
diagnostics.append(
|
|
_node_field_error(node, "select.field", "Selected fields need a source column.", "fields")
|
|
)
|
|
break
|
|
elif node.type == "aggregate":
|
|
group_by = config.get("group_by", [])
|
|
aggregates = config.get("aggregates")
|
|
if not isinstance(group_by, list) or any(not _non_empty_text(item) for item in group_by):
|
|
diagnostics.append(
|
|
_node_field_error(node, "aggregate.group_by", "Group-by columns must be named.", "group_by")
|
|
)
|
|
if not isinstance(aggregates, list) or not aggregates:
|
|
diagnostics.append(
|
|
_node_field_error(node, "aggregate.required", "Add at least one aggregate.", "aggregates")
|
|
)
|
|
else:
|
|
for aggregate in aggregates:
|
|
if not isinstance(aggregate, dict) or aggregate.get("function") not in AGGREGATE_FUNCTIONS:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"aggregate.function",
|
|
"Choose COUNT, SUM, AVG, MIN, or MAX.",
|
|
"aggregates",
|
|
)
|
|
)
|
|
break
|
|
if aggregate.get("function") != "count" and not _non_empty_text(aggregate.get("column")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"aggregate.column",
|
|
"This aggregate needs a source column.",
|
|
"aggregates",
|
|
)
|
|
)
|
|
break
|
|
if not _non_empty_text(aggregate.get("alias")):
|
|
diagnostics.append(
|
|
_node_field_error(node, "aggregate.alias", "Every aggregate needs an alias.", "aggregates")
|
|
)
|
|
break
|
|
elif node.type == "derive":
|
|
if not _non_empty_text(config.get("target_column")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"derive.target_column",
|
|
"Choose an output column.",
|
|
"target_column",
|
|
)
|
|
)
|
|
operation = config.get("operation")
|
|
if operation not in DERIVE_OPERATIONS:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"derive.operation",
|
|
"Choose a supported derive operation.",
|
|
"operation",
|
|
)
|
|
)
|
|
source_columns = config.get("source_columns")
|
|
if (
|
|
not isinstance(source_columns, list)
|
|
or not source_columns
|
|
or any(not _non_empty_text(item) for item in source_columns)
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"derive.source_columns",
|
|
"Choose at least one source column.",
|
|
"source_columns",
|
|
)
|
|
)
|
|
if operation in {"copy", "upper", "lower", "trim"} and isinstance(source_columns, list) and len(source_columns) != 1:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"derive.source_count",
|
|
"This operation requires exactly one source column.",
|
|
"source_columns",
|
|
)
|
|
)
|
|
if operation in {"add", "subtract", "multiply", "divide"} and isinstance(source_columns, list) and len(source_columns) != 2:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"derive.numeric_source_count",
|
|
"Numeric operations require exactly two source columns.",
|
|
"source_columns",
|
|
)
|
|
)
|
|
elif node.type == "expression":
|
|
if not _non_empty_text(config.get("target_column")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"expression.target_column",
|
|
"Choose an output column.",
|
|
"target_column",
|
|
)
|
|
)
|
|
if not _non_empty_text(config.get("expression")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"expression.required",
|
|
"Enter an expression.",
|
|
"expression",
|
|
)
|
|
)
|
|
if str(config.get("result_type") or "unknown") not in {
|
|
"unknown",
|
|
*DATA_TYPES,
|
|
}:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"expression.result_type",
|
|
"Choose a supported expression result type.",
|
|
"result_type",
|
|
)
|
|
)
|
|
elif node.type == "convert":
|
|
for field_name, message in (
|
|
("source_column", "Choose a source column."),
|
|
("target_column", "Choose an output column."),
|
|
):
|
|
if not _non_empty_text(config.get(field_name)):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
f"convert.{field_name}",
|
|
message,
|
|
field_name,
|
|
)
|
|
)
|
|
if config.get("target_type") not in DATA_TYPES:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"convert.target_type",
|
|
"Choose a supported conversion type.",
|
|
"target_type",
|
|
)
|
|
)
|
|
if config.get("on_error", "fail") not in {"fail", "null", "keep"}:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"convert.on_error",
|
|
"Choose how conversion errors are handled.",
|
|
"on_error",
|
|
)
|
|
)
|
|
elif node.type == "replace":
|
|
for field_name, message in (
|
|
("source_column", "Choose a source column."),
|
|
("target_column", "Choose an output column."),
|
|
):
|
|
if not _non_empty_text(config.get(field_name)):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
f"replace.{field_name}",
|
|
message,
|
|
field_name,
|
|
)
|
|
)
|
|
if config.get("mode", "exact") not in {"exact", "text"}:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"replace.mode",
|
|
"Choose exact-value or text replacement.",
|
|
"mode",
|
|
)
|
|
)
|
|
elif node.type == "sort":
|
|
fields = config.get("fields")
|
|
if not isinstance(fields, list) or not fields:
|
|
diagnostics.append(_node_field_error(node, "sort.fields", "Add at least one sort field.", "fields"))
|
|
elif any(
|
|
not isinstance(item, dict)
|
|
or not _non_empty_text(item.get("column"))
|
|
or item.get("direction", "asc") not in {"asc", "desc"}
|
|
for item in fields
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(node, "sort.field", "Sort fields need a column and direction.", "fields")
|
|
)
|
|
elif node.type == "limit":
|
|
count = config.get("count")
|
|
if not isinstance(count, int) or isinstance(count, bool) or not 1 <= count <= 100_000:
|
|
diagnostics.append(
|
|
_node_field_error(node, "limit.count", "Limit must be between 1 and 100,000.", "count")
|
|
)
|
|
elif node.type == "quality.rules":
|
|
rules = config.get("rules")
|
|
if not isinstance(rules, list) or not rules or len(rules) > 100:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.rules",
|
|
"Add between one and 100 quality rules.",
|
|
"rules",
|
|
)
|
|
)
|
|
else:
|
|
rule_ids: list[str] = []
|
|
for rule in rules:
|
|
if not isinstance(rule, dict):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.rule_shape",
|
|
"Every quality rule must be an object.",
|
|
"rules",
|
|
)
|
|
)
|
|
break
|
|
rule_id = str(rule.get("id") or "").strip()
|
|
rule_ids.append(rule_id)
|
|
operator = rule.get("operator")
|
|
if not rule_id or not _non_empty_text(rule.get("column")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.rule_identity",
|
|
"Quality rules need an ID and column.",
|
|
"rules",
|
|
)
|
|
)
|
|
break
|
|
if operator not in QUALITY_OPERATORS:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.operator",
|
|
"Choose a supported quality operator.",
|
|
"rules",
|
|
)
|
|
)
|
|
break
|
|
if operator in {"type", "min", "max"} and "value" not in rule:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.value",
|
|
"This quality rule requires a value.",
|
|
"rules",
|
|
)
|
|
)
|
|
break
|
|
if operator == "allowed" and not isinstance(rule.get("values"), list):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.values",
|
|
"Allowed-value rules require a list of values.",
|
|
"rules",
|
|
)
|
|
)
|
|
break
|
|
duplicates = sorted(
|
|
{rule_id for rule_id in rule_ids if rule_ids.count(rule_id) > 1}
|
|
)
|
|
if duplicates:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.duplicate_id",
|
|
f"Quality rule IDs must be unique: {', '.join(duplicates)}.",
|
|
"rules",
|
|
)
|
|
)
|
|
if config.get("action", "annotate") not in {"annotate", "drop", "fail"}:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"quality.action",
|
|
"Choose how invalid rows are handled.",
|
|
"action",
|
|
)
|
|
)
|
|
elif node.type == "reconcile.compare":
|
|
left_keys = config.get("left_keys")
|
|
right_keys = config.get("right_keys")
|
|
if (
|
|
not isinstance(left_keys, list)
|
|
or not left_keys
|
|
or any(not _non_empty_text(item) for item in left_keys)
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"reconcile.left_keys",
|
|
"Add at least one expected-table key.",
|
|
"left_keys",
|
|
)
|
|
)
|
|
if (
|
|
not isinstance(right_keys, list)
|
|
or not right_keys
|
|
or any(not _non_empty_text(item) for item in right_keys)
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"reconcile.right_keys",
|
|
"Add at least one observed-table key.",
|
|
"right_keys",
|
|
)
|
|
)
|
|
if (
|
|
isinstance(left_keys, list)
|
|
and isinstance(right_keys, list)
|
|
and len(left_keys) != len(right_keys)
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"reconcile.key_count",
|
|
"Expected and observed keys must have the same length.",
|
|
"right_keys",
|
|
)
|
|
)
|
|
prefix = config.get("right_prefix", "observed_")
|
|
if (
|
|
not _non_empty_text(prefix)
|
|
or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*_", str(prefix)) is None
|
|
):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"reconcile.right_prefix",
|
|
"Enter an identifier prefix ending in an underscore.",
|
|
"right_prefix",
|
|
)
|
|
)
|
|
elif node.type == "subflow":
|
|
if not _non_empty_text(config.get("template_ref")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"subflow.template_ref",
|
|
"Choose a reusable template.",
|
|
"template_ref",
|
|
)
|
|
)
|
|
if not _non_empty_text(config.get("template_version")):
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"subflow.template_version",
|
|
"Pin a template version.",
|
|
"template_version",
|
|
)
|
|
)
|
|
parameters = config.get("parameters")
|
|
if not isinstance(parameters, dict) or len(parameters) > 100:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"subflow.parameters",
|
|
"Subflow parameters must be an object with at most 100 entries.",
|
|
"parameters",
|
|
)
|
|
)
|
|
try:
|
|
nested_payload = substitute_parameters(
|
|
config.get("graph"),
|
|
parameters if isinstance(parameters, dict) else {},
|
|
)
|
|
nested = PipelineGraph.model_validate(nested_payload)
|
|
except (SubflowParameterError, ValueError) as exc:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"subflow.graph",
|
|
f"Pinned subflow graph is invalid: {exc}",
|
|
"graph",
|
|
)
|
|
)
|
|
else:
|
|
bound_inputs = [
|
|
item
|
|
for item in nested.nodes
|
|
if item.type == "source.inline"
|
|
and item.config.get("input_binding") is True
|
|
]
|
|
if len(bound_inputs) != 1:
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"subflow.input_binding",
|
|
"Pinned subflows need exactly one inline input binding.",
|
|
"graph",
|
|
)
|
|
)
|
|
for nested_diagnostic in validate_graph(nested):
|
|
if nested_diagnostic.severity == "error":
|
|
diagnostics.append(
|
|
_node_field_error(
|
|
node,
|
|
"subflow.graph_validation",
|
|
f"Pinned subflow: {nested_diagnostic.message}",
|
|
"graph",
|
|
)
|
|
)
|
|
break
|
|
return diagnostics
|
|
|
|
|
|
def _non_empty_text(value: object) -> bool:
|
|
return isinstance(value, str) and bool(value.strip())
|
|
|
|
|
|
def _node_field_error(node: GraphNode, code: str, message: str, field: str) -> DataflowDiagnostic:
|
|
return _error(code, message, node_id=node.id, field=field)
|
|
|
|
|
|
def _error(
|
|
code: str,
|
|
message: str,
|
|
*,
|
|
node_id: str | None = None,
|
|
field: str | None = None,
|
|
) -> DataflowDiagnostic:
|
|
return DataflowDiagnostic(
|
|
severity="error",
|
|
code=code,
|
|
message=message,
|
|
node_id=node_id,
|
|
field=field,
|
|
)
|
|
|
|
|
|
def _warning(
|
|
code: str,
|
|
message: str,
|
|
*,
|
|
node_id: str | None = None,
|
|
field: str | None = None,
|
|
) -> DataflowDiagnostic:
|
|
return DataflowDiagnostic(
|
|
severity="warning",
|
|
code=code,
|
|
message=message,
|
|
node_id=node_id,
|
|
field=field,
|
|
)
|
|
|
|
|
|
def _register_config_validators() -> None:
|
|
for node_type in SUPPORTED_NODE_TYPES:
|
|
if OPERATOR_REGISTRY.config_validator(node_type) is None:
|
|
OPERATOR_REGISTRY.register_config_validator(
|
|
node_type,
|
|
_validate_node_config,
|
|
)
|
|
|
|
|
|
_register_config_validators()
|
|
|
|
|
|
__all__ = [
|
|
"AGGREGATE_FUNCTIONS",
|
|
"DERIVE_OPERATIONS",
|
|
"FILTER_OPERATORS",
|
|
"JOIN_TYPES",
|
|
"SUPPORTED_NODE_TYPES",
|
|
"canonical_graph_payload",
|
|
"definition_hash",
|
|
"graph_inputs_by_port",
|
|
"topological_order",
|
|
"validate_graph",
|
|
]
|