Expand governed Dataflow editor and node library
This commit is contained in:
@@ -2,28 +2,35 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphNode, PipelineGraph
|
||||
from govoplan_dataflow.backend.node_library import NODE_TYPES, node_definition
|
||||
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphEdge, GraphNode, PipelineGraph
|
||||
|
||||
|
||||
SUPPORTED_NODE_TYPES = frozenset(
|
||||
{
|
||||
"source.inline",
|
||||
"source.reference",
|
||||
"filter",
|
||||
"select",
|
||||
"aggregate",
|
||||
"sort",
|
||||
"limit",
|
||||
"output",
|
||||
}
|
||||
)
|
||||
SUPPORTED_NODE_TYPES = frozenset(NODE_TYPES)
|
||||
FILTER_OPERATORS = frozenset(
|
||||
{"eq", "ne", "gt", "gte", "lt", "lte", "contains", "is_null", "not_null"}
|
||||
)
|
||||
AGGREGATE_FUNCTIONS = frozenset({"count", "sum", "avg", "min", "max"})
|
||||
DERIVE_OPERATIONS = frozenset(
|
||||
{
|
||||
"copy",
|
||||
"upper",
|
||||
"lower",
|
||||
"trim",
|
||||
"concat",
|
||||
"coalesce",
|
||||
"add",
|
||||
"subtract",
|
||||
"multiply",
|
||||
"divide",
|
||||
}
|
||||
)
|
||||
JOIN_TYPES = frozenset({"inner", "left", "right", "full"})
|
||||
|
||||
|
||||
def canonical_graph_payload(graph: PipelineGraph) -> dict[str, Any]:
|
||||
@@ -49,7 +56,7 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
if len(edge_ids) != len(graph.edges):
|
||||
diagnostics.append(_error("graph.duplicate_edge", "Edge identifiers must be unique."))
|
||||
|
||||
incoming: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
incoming: dict[str, list[GraphEdge]] = {node_id: [] for node_id in nodes}
|
||||
outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
for edge in graph.edges:
|
||||
if edge.source not in nodes:
|
||||
@@ -67,8 +74,32 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
_error("edge.self_reference", "A node cannot connect to itself.", node_id=edge.source)
|
||||
)
|
||||
continue
|
||||
source_definition = node_definition(nodes[edge.source].type)
|
||||
target_definition = node_definition(nodes[edge.target].type)
|
||||
if source_definition and edge.source_port not in {
|
||||
port.id for port in source_definition.output_ports
|
||||
}:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"edge.unknown_source_port",
|
||||
f"Node {edge.source!r} has no output port {edge.source_port!r}.",
|
||||
node_id=edge.source,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if target_definition and edge.target_port not in {
|
||||
port.id for port in target_definition.input_ports
|
||||
}:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"edge.unknown_target_port",
|
||||
f"Node {edge.target!r} has no input port {edge.target_port!r}.",
|
||||
node_id=edge.target,
|
||||
)
|
||||
)
|
||||
continue
|
||||
outgoing[edge.source].append(edge.target)
|
||||
incoming[edge.target].append(edge.source)
|
||||
incoming[edge.target].append(edge)
|
||||
|
||||
if not graph.nodes:
|
||||
diagnostics.append(_error("graph.empty", "Add a source and an output before saving the pipeline."))
|
||||
@@ -76,11 +107,34 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
|
||||
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
|
||||
output_nodes = [node for node in graph.nodes if node.type == "output"]
|
||||
if len(source_nodes) != 1:
|
||||
if not source_nodes:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"graph.source_count",
|
||||
"The first release supports exactly one source node per pipeline.",
|
||||
"A pipeline needs at least one source node.",
|
||||
)
|
||||
)
|
||||
elif len(source_nodes) > 10:
|
||||
diagnostics.append(_error("graph.source_limit", "Pipelines are limited to ten sources."))
|
||||
source_names = [
|
||||
str(node.config.get("source_name", "")).strip()
|
||||
for node in source_nodes
|
||||
if _non_empty_text(node.config.get("source_name"))
|
||||
]
|
||||
duplicate_source_names = sorted(
|
||||
{
|
||||
name
|
||||
for name in source_names
|
||||
if sum(candidate.casefold() == name.casefold() for candidate in source_names) > 1
|
||||
},
|
||||
key=str.casefold,
|
||||
)
|
||||
if duplicate_source_names:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"source.duplicate_name",
|
||||
"Logical source names must be unique: "
|
||||
f"{', '.join(duplicate_source_names)}.",
|
||||
)
|
||||
)
|
||||
if len(output_nodes) != 1:
|
||||
@@ -99,31 +153,50 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
)
|
||||
)
|
||||
continue
|
||||
if node.type.startswith("source."):
|
||||
if incoming.get(node.id):
|
||||
diagnostics.append(
|
||||
_error("node.source_has_input", "Source nodes cannot have incoming edges.", node_id=node.id)
|
||||
)
|
||||
elif len(incoming.get(node.id, [])) != 1:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.input_count",
|
||||
"This transform requires exactly one incoming edge.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
definition = node_definition(node.type)
|
||||
node_edges = incoming.get(node.id, [])
|
||||
if definition is not None:
|
||||
for port in definition.input_ports:
|
||||
connections = [
|
||||
edge
|
||||
for edge in node_edges
|
||||
if edge.target_port == port.id
|
||||
]
|
||||
minimum = port.minimum_connections if port.required else 0
|
||||
if len(connections) < minimum:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.input_required",
|
||||
f"{definition.label} requires {port.label.lower()} input.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
if not port.multiple and len(connections) > 1:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"node.input_multiple",
|
||||
f"{port.label} accepts only one connection.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
diagnostics.extend(_validate_node_config(node))
|
||||
|
||||
ordered, cyclic = topological_order(graph)
|
||||
if cyclic:
|
||||
diagnostics.append(_error("graph.cycle", "Pipeline edges must form an acyclic graph."))
|
||||
elif source_nodes and output_nodes:
|
||||
reachable = _reachable_from(source_nodes[0].id, outgoing)
|
||||
reachable: set[str] = set()
|
||||
for source in source_nodes:
|
||||
reachable.update(_reachable_from(source.id, outgoing))
|
||||
if len(reachable) != len(nodes):
|
||||
diagnostics.append(
|
||||
_error("graph.disconnected", "Every node must be connected to the pipeline source.")
|
||||
_error("graph.disconnected", "Every node must be connected to a pipeline source.")
|
||||
)
|
||||
reaches_output = _reachable_from(output_nodes[0].id, incoming)
|
||||
reverse_adjacency = {
|
||||
node_id: [edge.source for edge in edges]
|
||||
for node_id, edges in incoming.items()
|
||||
}
|
||||
reaches_output = _reachable_from(output_nodes[0].id, reverse_adjacency)
|
||||
if len(reaches_output) != len(nodes):
|
||||
diagnostics.append(
|
||||
_error("graph.dead_end", "Every node must lead to the pipeline output.")
|
||||
@@ -132,6 +205,7 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
diagnostics.append(
|
||||
_error("graph.output_not_terminal", "The output node must be the terminal transform.")
|
||||
)
|
||||
diagnostics.extend(_validate_graph_schemas(graph, ordered=ordered))
|
||||
return diagnostics
|
||||
|
||||
|
||||
@@ -156,8 +230,11 @@ def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]:
|
||||
return ordered, len(ordered) != len(node_ids)
|
||||
|
||||
|
||||
def graph_input_map(graph: PipelineGraph) -> dict[str, str]:
|
||||
return {edge.target: edge.source for edge in graph.edges}
|
||||
def graph_inputs_by_port(graph: PipelineGraph) -> dict[str, dict[str, list[str]]]:
|
||||
result: dict[str, dict[str, list[str]]] = {}
|
||||
for edge in graph.edges:
|
||||
result.setdefault(edge.target, {}).setdefault(edge.target_port, []).append(edge.source)
|
||||
return result
|
||||
|
||||
|
||||
def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]:
|
||||
@@ -172,6 +249,306 @@ def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]:
|
||||
return seen
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SchemaState:
|
||||
columns: frozenset[str]
|
||||
open: bool = False
|
||||
|
||||
def knows(self, column: str) -> bool:
|
||||
return self.open or column in self.columns
|
||||
|
||||
|
||||
def _validate_graph_schemas(
|
||||
graph: PipelineGraph,
|
||||
*,
|
||||
ordered: list[str],
|
||||
) -> list[DataflowDiagnostic]:
|
||||
diagnostics: list[DataflowDiagnostic] = []
|
||||
node_by_id = {node.id: node for node in graph.nodes}
|
||||
inputs = graph_inputs_by_port(graph)
|
||||
schemas: dict[str, _SchemaState] = {}
|
||||
for node_id in ordered:
|
||||
node = node_by_id[node_id]
|
||||
node_inputs = inputs.get(node.id, {})
|
||||
input_states = [
|
||||
schemas[source_id]
|
||||
for port_sources in node_inputs.values()
|
||||
for source_id in port_sources
|
||||
if source_id in schemas
|
||||
]
|
||||
input_state = input_states[0] if input_states else _SchemaState(frozenset(), open=True)
|
||||
if node.type == "source.inline":
|
||||
rows = node.config.get("rows")
|
||||
columns = {
|
||||
str(column)
|
||||
for row in rows if isinstance(rows, list) and isinstance(row, dict)
|
||||
for column in row
|
||||
} if isinstance(rows, list) else set()
|
||||
schemas[node.id] = _SchemaState(
|
||||
frozenset(columns),
|
||||
open=not columns,
|
||||
)
|
||||
continue
|
||||
if node.type == "source.reference":
|
||||
columns = _configured_source_columns(node.config.get("source_columns"))
|
||||
schemas[node.id] = _SchemaState(
|
||||
frozenset(columns),
|
||||
open=not columns,
|
||||
)
|
||||
continue
|
||||
if node.type == "combine.union":
|
||||
if len(input_states) > 1:
|
||||
closed_shapes = {
|
||||
state.columns
|
||||
for state in input_states
|
||||
if not state.open
|
||||
}
|
||||
if len(closed_shapes) > 1:
|
||||
diagnostics.append(
|
||||
_warning(
|
||||
"union.schema_mismatch",
|
||||
"Appended inputs use different columns; missing values will be null.",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
schemas[node.id] = _SchemaState(
|
||||
frozenset().union(*(state.columns for state in input_states)),
|
||||
open=any(state.open for state in input_states),
|
||||
)
|
||||
continue
|
||||
if node.type == "combine.join":
|
||||
left_state = _port_schema(node_inputs, schemas, "left")
|
||||
right_state = _port_schema(node_inputs, schemas, "right")
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=left_state,
|
||||
columns=node.config.get("left_keys"),
|
||||
field="left_keys",
|
||||
)
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=right_state,
|
||||
columns=node.config.get("right_keys"),
|
||||
field="right_keys",
|
||||
)
|
||||
prefix = str(node.config.get("right_prefix", "right_"))
|
||||
prefixed_right = {f"{prefix}{column}" for column in right_state.columns}
|
||||
collisions = left_state.columns & prefixed_right
|
||||
if collisions:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"join.output_collision",
|
||||
f"Join output columns collide: {', '.join(sorted(collisions))}.",
|
||||
node_id=node.id,
|
||||
field="right_prefix",
|
||||
)
|
||||
)
|
||||
schemas[node.id] = _SchemaState(
|
||||
frozenset(left_state.columns | prefixed_right),
|
||||
open=left_state.open or right_state.open,
|
||||
)
|
||||
continue
|
||||
if node.type == "filter":
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=input_state,
|
||||
columns=[node.config.get("column")],
|
||||
field="column",
|
||||
)
|
||||
schemas[node.id] = input_state
|
||||
continue
|
||||
if node.type == "distinct":
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=input_state,
|
||||
columns=node.config.get("columns"),
|
||||
field="columns",
|
||||
)
|
||||
schemas[node.id] = input_state
|
||||
continue
|
||||
if node.type == "select":
|
||||
fields = node.config.get("fields")
|
||||
selected_columns: list[str] = []
|
||||
output_columns: list[str] = []
|
||||
if isinstance(fields, list):
|
||||
for field in fields:
|
||||
if isinstance(field, str):
|
||||
selected_columns.append(field)
|
||||
output_columns.append(field)
|
||||
elif isinstance(field, dict):
|
||||
column = field.get("column")
|
||||
alias = field.get("alias") or column
|
||||
if isinstance(column, str):
|
||||
selected_columns.append(column)
|
||||
if isinstance(alias, str):
|
||||
output_columns.append(alias)
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=input_state,
|
||||
columns=selected_columns,
|
||||
field="fields",
|
||||
)
|
||||
_validate_output_names(
|
||||
diagnostics,
|
||||
node=node,
|
||||
columns=output_columns,
|
||||
field="fields",
|
||||
)
|
||||
schemas[node.id] = _SchemaState(frozenset(output_columns))
|
||||
continue
|
||||
if node.type == "derive":
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=input_state,
|
||||
columns=node.config.get("source_columns"),
|
||||
field="source_columns",
|
||||
)
|
||||
target = node.config.get("target_column")
|
||||
if isinstance(target, str) and target:
|
||||
if target in input_state.columns:
|
||||
diagnostics.append(
|
||||
_warning(
|
||||
"derive.overwrites_column",
|
||||
f"Derived column {target!r} replaces an existing value.",
|
||||
node_id=node.id,
|
||||
field="target_column",
|
||||
)
|
||||
)
|
||||
schemas[node.id] = _SchemaState(
|
||||
input_state.columns | frozenset((target,)),
|
||||
open=input_state.open,
|
||||
)
|
||||
else:
|
||||
schemas[node.id] = input_state
|
||||
continue
|
||||
if node.type == "aggregate":
|
||||
group_by = _text_items(node.config.get("group_by"))
|
||||
aggregates = node.config.get("aggregates")
|
||||
aggregate_columns = [
|
||||
str(item.get("column"))
|
||||
for item in aggregates
|
||||
if isinstance(aggregates, list)
|
||||
and isinstance(item, dict)
|
||||
and item.get("column") not in (None, "", "*")
|
||||
] if isinstance(aggregates, list) else []
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=input_state,
|
||||
columns=[*group_by, *aggregate_columns],
|
||||
field="aggregates",
|
||||
)
|
||||
aliases = [
|
||||
str(item.get("alias"))
|
||||
for item in aggregates
|
||||
if isinstance(aggregates, list)
|
||||
and isinstance(item, dict)
|
||||
and item.get("alias")
|
||||
] if isinstance(aggregates, list) else []
|
||||
output_columns = [*group_by, *aliases]
|
||||
_validate_output_names(
|
||||
diagnostics,
|
||||
node=node,
|
||||
columns=output_columns,
|
||||
field="aggregates",
|
||||
)
|
||||
schemas[node.id] = _SchemaState(frozenset(output_columns))
|
||||
continue
|
||||
if node.type == "sort":
|
||||
fields = node.config.get("fields")
|
||||
columns = [
|
||||
str(item.get("column"))
|
||||
for item in fields
|
||||
if isinstance(fields, list)
|
||||
and isinstance(item, dict)
|
||||
and item.get("column")
|
||||
] if isinstance(fields, list) else []
|
||||
_validate_columns(
|
||||
diagnostics,
|
||||
node=node,
|
||||
state=input_state,
|
||||
columns=columns,
|
||||
field="fields",
|
||||
)
|
||||
schemas[node.id] = input_state
|
||||
continue
|
||||
schemas[node.id] = input_state
|
||||
return diagnostics
|
||||
|
||||
|
||||
def _configured_source_columns(value: object) -> set[str]:
|
||||
if not isinstance(value, list):
|
||||
return set()
|
||||
return {
|
||||
item if isinstance(item, str) else str(item.get("name"))
|
||||
for item in value
|
||||
if (
|
||||
isinstance(item, str) and item
|
||||
or isinstance(item, dict) and item.get("name")
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _port_schema(
|
||||
inputs: dict[str, list[str]],
|
||||
schemas: dict[str, _SchemaState],
|
||||
port: str,
|
||||
) -> _SchemaState:
|
||||
source_ids = inputs.get(port, [])
|
||||
return schemas.get(source_ids[0], _SchemaState(frozenset(), open=True)) if source_ids else _SchemaState(frozenset(), open=True)
|
||||
|
||||
|
||||
def _validate_columns(
|
||||
diagnostics: list[DataflowDiagnostic],
|
||||
*,
|
||||
node: GraphNode,
|
||||
state: _SchemaState,
|
||||
columns: object,
|
||||
field: str,
|
||||
) -> None:
|
||||
for column in _text_items(columns):
|
||||
if not state.knows(column):
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"schema.unknown_column",
|
||||
f"Column {column!r} is not available at this node.",
|
||||
node_id=node.id,
|
||||
field=field,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _validate_output_names(
|
||||
diagnostics: list[DataflowDiagnostic],
|
||||
*,
|
||||
node: GraphNode,
|
||||
columns: list[str],
|
||||
field: str,
|
||||
) -> None:
|
||||
duplicates = sorted({column for column in columns if columns.count(column) > 1})
|
||||
if duplicates:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"schema.duplicate_output",
|
||||
f"Output column names must be unique: {', '.join(duplicates)}.",
|
||||
node_id=node.id,
|
||||
field=field,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _text_items(value: object) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, str) and item]
|
||||
|
||||
|
||||
def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
||||
config = node.config
|
||||
diagnostics: list[DataflowDiagnostic] = []
|
||||
@@ -186,7 +563,35 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
||||
field="source_name",
|
||||
)
|
||||
)
|
||||
elif re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", source_name.strip()) is None:
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"source.name_invalid",
|
||||
"Logical source names must be SQL identifiers, such as monthly_cases.",
|
||||
node_id=node.id,
|
||||
field="source_name",
|
||||
)
|
||||
)
|
||||
if node.type == "source.reference":
|
||||
if not _non_empty_text(config.get("source_ref")):
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"source.reference_required",
|
||||
"Choose a connector source.",
|
||||
node_id=node.id,
|
||||
field="source_ref",
|
||||
)
|
||||
)
|
||||
expected_fingerprint = config.get("expected_fingerprint")
|
||||
if expected_fingerprint is not None and not isinstance(expected_fingerprint, str):
|
||||
diagnostics.append(
|
||||
_error(
|
||||
"source.fingerprint",
|
||||
"The expected source fingerprint must be text.",
|
||||
node_id=node.id,
|
||||
field="expected_fingerprint",
|
||||
)
|
||||
)
|
||||
return diagnostics
|
||||
rows = config.get("rows")
|
||||
if not isinstance(rows, list):
|
||||
@@ -215,6 +620,72 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
||||
)
|
||||
if operator not in {"is_null", "not_null"} and "value" not in config:
|
||||
diagnostics.append(_node_field_error(node, "filter.value", "Enter a comparison value.", "value"))
|
||||
elif node.type == "distinct":
|
||||
columns = config.get("columns", [])
|
||||
if not isinstance(columns, list) or any(not _non_empty_text(item) for item in columns):
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"distinct.columns",
|
||||
"Distinct key columns must be named.",
|
||||
"columns",
|
||||
)
|
||||
)
|
||||
elif node.type == "combine.union":
|
||||
if config.get("mode", "all") not in {"all", "distinct"}:
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"union.mode",
|
||||
"Choose whether duplicate rows are kept or removed.",
|
||||
"mode",
|
||||
)
|
||||
)
|
||||
elif node.type == "combine.join":
|
||||
if config.get("join_type", "inner") not in JOIN_TYPES:
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "join.type", "Choose a supported join type.", "join_type")
|
||||
)
|
||||
left_keys = config.get("left_keys")
|
||||
right_keys = config.get("right_keys")
|
||||
if (
|
||||
not isinstance(left_keys, list)
|
||||
or not left_keys
|
||||
or any(not _non_empty_text(item) for item in left_keys)
|
||||
):
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "join.left_keys", "Add at least one left key.", "left_keys")
|
||||
)
|
||||
if (
|
||||
not isinstance(right_keys, list)
|
||||
or not right_keys
|
||||
or any(not _non_empty_text(item) for item in right_keys)
|
||||
):
|
||||
diagnostics.append(
|
||||
_node_field_error(node, "join.right_keys", "Add at least one right key.", "right_keys")
|
||||
)
|
||||
if isinstance(left_keys, list) and isinstance(right_keys, list) and len(left_keys) != len(right_keys):
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"join.key_count",
|
||||
"Left and right joins need the same number of keys.",
|
||||
"right_keys",
|
||||
)
|
||||
)
|
||||
right_prefix = config.get("right_prefix", "right_")
|
||||
if (
|
||||
not _non_empty_text(right_prefix)
|
||||
or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*_", str(right_prefix)) is None
|
||||
):
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"join.right_prefix",
|
||||
"Enter an identifier prefix ending in an underscore, such as right_.",
|
||||
"right_prefix",
|
||||
)
|
||||
)
|
||||
elif node.type == "select":
|
||||
fields = config.get("fields")
|
||||
if not isinstance(fields, list) or not fields:
|
||||
@@ -267,6 +738,58 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
|
||||
_node_field_error(node, "aggregate.alias", "Every aggregate needs an alias.", "aggregates")
|
||||
)
|
||||
break
|
||||
elif node.type == "derive":
|
||||
if not _non_empty_text(config.get("target_column")):
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"derive.target_column",
|
||||
"Choose an output column.",
|
||||
"target_column",
|
||||
)
|
||||
)
|
||||
operation = config.get("operation")
|
||||
if operation not in DERIVE_OPERATIONS:
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"derive.operation",
|
||||
"Choose a supported derive operation.",
|
||||
"operation",
|
||||
)
|
||||
)
|
||||
source_columns = config.get("source_columns")
|
||||
if (
|
||||
not isinstance(source_columns, list)
|
||||
or not source_columns
|
||||
or any(not _non_empty_text(item) for item in source_columns)
|
||||
):
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"derive.source_columns",
|
||||
"Choose at least one source column.",
|
||||
"source_columns",
|
||||
)
|
||||
)
|
||||
if operation in {"copy", "upper", "lower", "trim"} and isinstance(source_columns, list) and len(source_columns) != 1:
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"derive.source_count",
|
||||
"This operation requires exactly one source column.",
|
||||
"source_columns",
|
||||
)
|
||||
)
|
||||
if operation in {"add", "subtract", "multiply", "divide"} and isinstance(source_columns, list) and len(source_columns) != 2:
|
||||
diagnostics.append(
|
||||
_node_field_error(
|
||||
node,
|
||||
"derive.numeric_source_count",
|
||||
"Numeric operations require exactly two source columns.",
|
||||
"source_columns",
|
||||
)
|
||||
)
|
||||
elif node.type == "sort":
|
||||
fields = config.get("fields")
|
||||
if not isinstance(fields, list) or not fields:
|
||||
@@ -313,13 +836,31 @@ def _error(
|
||||
)
|
||||
|
||||
|
||||
def _warning(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
node_id: str | None = None,
|
||||
field: str | None = None,
|
||||
) -> DataflowDiagnostic:
|
||||
return DataflowDiagnostic(
|
||||
severity="warning",
|
||||
code=code,
|
||||
message=message,
|
||||
node_id=node_id,
|
||||
field=field,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AGGREGATE_FUNCTIONS",
|
||||
"DERIVE_OPERATIONS",
|
||||
"FILTER_OPERATORS",
|
||||
"JOIN_TYPES",
|
||||
"SUPPORTED_NODE_TYPES",
|
||||
"canonical_graph_payload",
|
||||
"definition_hash",
|
||||
"graph_input_map",
|
||||
"graph_inputs_by_port",
|
||||
"topological_order",
|
||||
"validate_graph",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user