GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add extensible operators and golden flows

Browse Source
This commit is contained in:
zemion
2026-07-29 15:50:15 +02:00
parent 09c98087c5
commit 946202ef01
27 changed files with 3584 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_dataflow/backend/graph.py
+709 -20
View File
@@ -4,7 +4,7 @@ import hashlib
import json
import re
from collections import deque
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any
from govoplan_core.core.definition_graphs import (
@@ -17,6 +17,16 @@ from govoplan_dataflow.backend.node_library import (
NODE_TYPES,
node_definition,
)
from govoplan_dataflow.backend.expressions import (
ExpressionError,
infer_expression_type,
parse_expression,
)
from govoplan_dataflow.backend.operator_registry import OPERATOR_REGISTRY
from govoplan_dataflow.backend.subflows import (
SubflowParameterError,
substitute_parameters,
)
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphEdge, GraphNode, PipelineGraph
@@ -40,6 +50,12 @@ DERIVE_OPERATIONS = frozenset(
}
)
JOIN_TYPES = frozenset({"inner", "left", "right", "full"})
DATA_TYPES = frozenset(
{"string", "integer", "number", "boolean", "date", "datetime"}
)
QUALITY_OPERATORS = frozenset(
{"not_null", "type", "min", "max", "allowed", "unique"}
)
def canonical_graph_payload(graph: PipelineGraph) -> dict[str, Any]:
@@ -191,7 +207,17 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
for node in graph.nodes:
if node.type not in SUPPORTED_NODE_TYPES:
continue
diagnostics.extend(_validate_node_config(node))
validator = OPERATOR_REGISTRY.config_validator(node.type)
if validator is None:
diagnostics.append(
_error(
"node.validator_missing",
f"Node type {node.type!r} has no registered validator.",
node_id=node.id,
)
)
continue
diagnostics.extend(validator(node))
ordered, cyclic = topological_order(graph)
if not cyclic and source_nodes and len(output_nodes) == 1:
@@ -341,10 +367,14 @@ def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]:
class _SchemaState:
columns: frozenset[str]
open: bool = False
types: dict[str, str] = field(default_factory=dict)
def knows(self, column: str) -> bool:
return self.open or column in self.columns
def type_of(self, column: str) -> str:
return self.types.get(column, "unknown")
def _validate_graph_schemas(
graph: PipelineGraph,
@@ -367,21 +397,11 @@ def _validate_graph_schemas(
input_state = input_states[0] if input_states else _SchemaState(frozenset(), open=True)
if node.type == "source.inline":
rows = node.config.get("rows")
columns = {
str(column)
for row in rows if isinstance(rows, list) and isinstance(row, dict)
for column in row
} if isinstance(rows, list) else set()
schemas[node.id] = _SchemaState(
frozenset(columns),
open=not columns,
)
schemas[node.id] = _inline_schema(rows)
continue
if node.type == "source.reference":
columns = _configured_source_columns(node.config.get("source_columns"))
schemas[node.id] = _SchemaState(
frozenset(columns),
open=not columns,
schemas[node.id] = _configured_source_schema(
node.config.get("source_columns")
)
continue
if node.type == "combine.union":
@@ -402,6 +422,7 @@ def _validate_graph_schemas(
schemas[node.id] = _SchemaState(
frozenset().union(*(state.columns for state in input_states)),
open=any(state.open for state in input_states),
types=_merged_schema_types(input_states),
)
continue
if node.type == "combine.join":
@@ -436,6 +457,13 @@ def _validate_graph_schemas(
schemas[node.id] = _SchemaState(
frozenset(left_state.columns | prefixed_right),
open=left_state.open or right_state.open,
types={
**left_state.types,
**{
f"{prefix}{column}": right_state.type_of(column)
for column in right_state.columns
},
},
)
continue
if node.type == "filter":
@@ -448,6 +476,22 @@ def _validate_graph_schemas(
)
schemas[node.id] = input_state
continue
if node.type == "filter.expression":
parsed = _parsed_node_expression(
diagnostics,
node=node,
field_name="expression",
)
if parsed is not None:
_validate_columns(
diagnostics,
node=node,
state=input_state,
columns=list(parsed.columns),
field="expression",
)
schemas[node.id] = input_state
continue
if node.type == "distinct":
_validate_columns(
diagnostics,
@@ -487,7 +531,17 @@ def _validate_graph_schemas(
columns=output_columns,
field="fields",
)
schemas[node.id] = _SchemaState(frozenset(output_columns))
schemas[node.id] = _SchemaState(
frozenset(output_columns),
types={
output: input_state.type_of(source)
for source, output in zip(
selected_columns,
output_columns,
strict=False,
)
},
)
continue
if node.type == "derive":
_validate_columns(
@@ -511,10 +565,95 @@ def _validate_graph_schemas(
schemas[node.id] = _SchemaState(
input_state.columns | frozenset((target,)),
open=input_state.open,
types={
**input_state.types,
target: _derive_result_type(
str(node.config.get("operation") or ""),
[
input_state.type_of(column)
for column in _text_items(
node.config.get("source_columns")
)
],
),
},
)
else:
schemas[node.id] = input_state
continue
if node.type == "expression":
parsed = _parsed_node_expression(
diagnostics,
node=node,
field_name="expression",
)
target = str(node.config.get("target_column") or "")
result_type = "unknown"
if parsed is not None:
_validate_columns(
diagnostics,
node=node,
state=input_state,
columns=list(parsed.columns),
field="expression",
)
result_type = infer_expression_type(
parsed,
{
name: input_state.type_of(name) # type: ignore[dict-item]
for name in input_state.columns
},
)
expected = str(node.config.get("result_type") or "unknown")
if (
expected != "unknown"
and result_type not in {"unknown", "null", expected}
):
diagnostics.append(
_warning(
"expression.type_mismatch",
f"Expression infers {result_type}, not {expected}.",
node_id=node.id,
field="result_type",
)
)
schemas[node.id] = _SchemaState(
input_state.columns | ({target} if target else set()),
open=input_state.open,
types={
**input_state.types,
**(
{target: expected if expected != "unknown" else result_type}
if target
else {}
),
},
)
continue
if node.type in {"convert", "replace"}:
source = str(node.config.get("source_column") or "")
target = str(node.config.get("target_column") or "")
_validate_columns(
diagnostics,
node=node,
state=input_state,
columns=[source],
field="source_column",
)
target_type = (
str(node.config.get("target_type") or "unknown")
if node.type == "convert"
else input_state.type_of(source)
)
schemas[node.id] = _SchemaState(
input_state.columns | ({target} if target else set()),
open=input_state.open,
types={
**input_state.types,
**({target: target_type} if target else {}),
},
)
continue
if node.type == "aggregate":
group_by = _text_items(node.config.get("group_by"))
aggregates = node.config.get("aggregates")
@@ -546,7 +685,27 @@ def _validate_graph_schemas(
columns=output_columns,
field="aggregates",
)
schemas[node.id] = _SchemaState(frozenset(output_columns))
aggregate_types = {
str(item.get("alias")): (
"integer"
if item.get("function") == "count"
else input_state.type_of(str(item.get("column") or ""))
)
for item in aggregates
if isinstance(aggregates, list)
and isinstance(item, dict)
and item.get("alias")
} if isinstance(aggregates, list) else {}
schemas[node.id] = _SchemaState(
frozenset(output_columns),
types={
**{
column: input_state.type_of(column)
for column in group_by
},
**aggregate_types,
},
)
continue
if node.type == "sort":
fields = node.config.get("fields")
@@ -566,14 +725,124 @@ def _validate_graph_schemas(
)
schemas[node.id] = input_state
continue
if node.type == "quality.rules":
rules = node.config.get("rules")
_validate_columns(
diagnostics,
node=node,
state=input_state,
columns=[
str(rule.get("column") or "")
for rule in rules
if isinstance(rules, list) and isinstance(rule, dict)
] if isinstance(rules, list) else [],
field="rules",
)
if node.config.get("action", "annotate") == "annotate":
schemas[node.id] = _SchemaState(
input_state.columns
| frozenset(("_quality_valid", "_quality_errors")),
open=input_state.open,
types={
**input_state.types,
"_quality_valid": "boolean",
"_quality_errors": "array",
},
)
else:
schemas[node.id] = input_state
continue
if node.type == "reconcile.compare":
left_state = _port_schema(node_inputs, schemas, "left")
right_state = _port_schema(node_inputs, schemas, "right")
_validate_columns(
diagnostics,
node=node,
state=left_state,
columns=node.config.get("left_keys"),
field="left_keys",
)
_validate_columns(
diagnostics,
node=node,
state=right_state,
columns=node.config.get("right_keys"),
field="right_keys",
)
comparison_columns = node.config.get("compare_columns")
if isinstance(comparison_columns, list):
left_columns: list[str] = []
right_columns: list[str] = []
for item in comparison_columns:
if isinstance(item, str):
left_columns.append(item)
right_columns.append(item)
elif isinstance(item, dict):
left_columns.append(
str(item.get("left") or item.get("column") or "")
)
right_columns.append(
str(
item.get("right")
or item.get("left")
or item.get("column")
or ""
)
)
_validate_columns(
diagnostics,
node=node,
state=left_state,
columns=left_columns,
field="compare_columns",
)
_validate_columns(
diagnostics,
node=node,
state=right_state,
columns=right_columns,
field="compare_columns",
)
prefix = str(node.config.get("right_prefix") or "observed_")
schemas[node.id] = _SchemaState(
left_state.columns
| frozenset(f"{prefix}{column}" for column in right_state.columns)
| frozenset(
(
"_reconciliation_status",
"_reconciliation_differences",
)
),
open=left_state.open or right_state.open,
types={
**left_state.types,
**{
f"{prefix}{column}": right_state.type_of(column)
for column in right_state.columns
},
"_reconciliation_status": "string",
"_reconciliation_differences": "array",
},
)
continue
if node.type == "subflow":
output_schema = _configured_source_schema(
node.config.get("output_schema")
)
schemas[node.id] = (
output_schema
if output_schema.columns
else _SchemaState(frozenset(), open=True)
)
continue
schemas[node.id] = input_state
return diagnostics
def _configured_source_columns(value: object) -> set[str]:
def _configured_source_schema(value: object) -> _SchemaState:
if not isinstance(value, list):
return set()
return {
return _SchemaState(frozenset(), open=True)
columns = {
item if isinstance(item, str) else str(item.get("name"))
for item in value
if (
@@ -581,6 +850,104 @@ def _configured_source_columns(value: object) -> set[str]:
or isinstance(item, dict) and item.get("name")
)
}
types = {
str(item.get("name")): str(
item.get("data_type") or item.get("type") or "unknown"
)
for item in value
if isinstance(item, dict) and item.get("name")
}
return _SchemaState(
frozenset(columns),
open=not columns,
types=types,
)
def _inline_schema(value: object) -> _SchemaState:
if not isinstance(value, list):
return _SchemaState(frozenset(), open=True)
columns = {
str(column)
for row in value
if isinstance(row, dict)
for column in row
}
types: dict[str, str] = {}
for column in columns:
observed = {
_schema_value_type(row.get(column))
for row in value
if isinstance(row, dict) and row.get(column) is not None
}
types[column] = observed.pop() if len(observed) == 1 else "unknown"
return _SchemaState(
frozenset(columns),
open=not columns,
types=types,
)
def _schema_value_type(value: object) -> str:
if isinstance(value, bool):
return "boolean"
if isinstance(value, int):
return "integer"
if isinstance(value, float):
return "number"
if isinstance(value, str):
return "string"
if isinstance(value, list):
return "array"
if isinstance(value, dict):
return "object"
return "unknown"
def _merged_schema_types(states: list[_SchemaState]) -> dict[str, str]:
columns = frozenset().union(*(state.columns for state in states))
types: dict[str, str] = {}
for column in columns:
observed = {
state.type_of(column)
for state in states
if column in state.columns and state.type_of(column) != "unknown"
}
types[column] = observed.pop() if len(observed) == 1 else "unknown"
return types
def _derive_result_type(operation: str, source_types: list[str]) -> str:
if operation in {"upper", "lower", "trim", "concat"}:
return "string"
if operation in {"add", "subtract", "multiply", "divide"}:
return "number" if "number" in source_types or operation == "divide" else "integer"
if operation in {"copy", "coalesce"}:
concrete = {
item for item in source_types if item not in {"unknown", "null"}
}
return concrete.pop() if len(concrete) == 1 else "unknown"
return "unknown"
def _parsed_node_expression(
diagnostics: list[DataflowDiagnostic],
*,
node: GraphNode,
field_name: str,
):
try:
return parse_expression(str(node.config.get(field_name) or ""))
except ExpressionError as exc:
diagnostics.append(
_error(
"expression.invalid",
str(exc),
node_id=node.id,
field=field_name,
)
)
return None
def _port_schema(
@@ -708,6 +1075,16 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
)
if operator not in {"is_null", "not_null"} and "value" not in config:
diagnostics.append(_node_field_error(node, "filter.value", "Enter a comparison value.", "value"))
elif node.type == "filter.expression":
if not _non_empty_text(config.get("expression")):
diagnostics.append(
_node_field_error(
node,
"expression.required",
"Enter a filter expression.",
"expression",
)
)
elif node.type == "distinct":
columns = config.get("columns", [])
if not isinstance(columns, list) or any(not _non_empty_text(item) for item in columns):
@@ -878,6 +1255,92 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
"source_columns",
)
)
elif node.type == "expression":
if not _non_empty_text(config.get("target_column")):
diagnostics.append(
_node_field_error(
node,
"expression.target_column",
"Choose an output column.",
"target_column",
)
)
if not _non_empty_text(config.get("expression")):
diagnostics.append(
_node_field_error(
node,
"expression.required",
"Enter an expression.",
"expression",
)
)
if str(config.get("result_type") or "unknown") not in {
"unknown",
*DATA_TYPES,
}:
diagnostics.append(
_node_field_error(
node,
"expression.result_type",
"Choose a supported expression result type.",
"result_type",
)
)
elif node.type == "convert":
for field_name, message in (
("source_column", "Choose a source column."),
("target_column", "Choose an output column."),
):
if not _non_empty_text(config.get(field_name)):
diagnostics.append(
_node_field_error(
node,
f"convert.{field_name}",
message,
field_name,
)
)
if config.get("target_type") not in DATA_TYPES:
diagnostics.append(
_node_field_error(
node,
"convert.target_type",
"Choose a supported conversion type.",
"target_type",
)
)
if config.get("on_error", "fail") not in {"fail", "null", "keep"}:
diagnostics.append(
_node_field_error(
node,
"convert.on_error",
"Choose how conversion errors are handled.",
"on_error",
)
)
elif node.type == "replace":
for field_name, message in (
("source_column", "Choose a source column."),
("target_column", "Choose an output column."),
):
if not _non_empty_text(config.get(field_name)):
diagnostics.append(
_node_field_error(
node,
f"replace.{field_name}",
message,
field_name,
)
)
if config.get("mode", "exact") not in {"exact", "text"}:
diagnostics.append(
_node_field_error(
node,
"replace.mode",
"Choose exact-value or text replacement.",
"mode",
)
)
elif node.type == "sort":
fields = config.get("fields")
if not isinstance(fields, list) or not fields:
@@ -897,6 +1360,220 @@ def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
diagnostics.append(
_node_field_error(node, "limit.count", "Limit must be between 1 and 100,000.", "count")
)
elif node.type == "quality.rules":
rules = config.get("rules")
if not isinstance(rules, list) or not rules or len(rules) > 100:
diagnostics.append(
_node_field_error(
node,
"quality.rules",
"Add between one and 100 quality rules.",
"rules",
)
)
else:
rule_ids: list[str] = []
for rule in rules:
if not isinstance(rule, dict):
diagnostics.append(
_node_field_error(
node,
"quality.rule_shape",
"Every quality rule must be an object.",
"rules",
)
)
break
rule_id = str(rule.get("id") or "").strip()
rule_ids.append(rule_id)
operator = rule.get("operator")
if not rule_id or not _non_empty_text(rule.get("column")):
diagnostics.append(
_node_field_error(
node,
"quality.rule_identity",
"Quality rules need an ID and column.",
"rules",
)
)
break
if operator not in QUALITY_OPERATORS:
diagnostics.append(
_node_field_error(
node,
"quality.operator",
"Choose a supported quality operator.",
"rules",
)
)
break
if operator in {"type", "min", "max"} and "value" not in rule:
diagnostics.append(
_node_field_error(
node,
"quality.value",
"This quality rule requires a value.",
"rules",
)
)
break
if operator == "allowed" and not isinstance(rule.get("values"), list):
diagnostics.append(
_node_field_error(
node,
"quality.values",
"Allowed-value rules require a list of values.",
"rules",
)
)
break
duplicates = sorted(
{rule_id for rule_id in rule_ids if rule_ids.count(rule_id) > 1}
)
if duplicates:
diagnostics.append(
_node_field_error(
node,
"quality.duplicate_id",
f"Quality rule IDs must be unique: {', '.join(duplicates)}.",
"rules",
)
)
if config.get("action", "annotate") not in {"annotate", "drop", "fail"}:
diagnostics.append(
_node_field_error(
node,
"quality.action",
"Choose how invalid rows are handled.",
"action",
)
)
elif node.type == "reconcile.compare":
left_keys = config.get("left_keys")
right_keys = config.get("right_keys")
if (
not isinstance(left_keys, list)
or not left_keys
or any(not _non_empty_text(item) for item in left_keys)
):
diagnostics.append(
_node_field_error(
node,
"reconcile.left_keys",
"Add at least one expected-table key.",
"left_keys",
)
)
if (
not isinstance(right_keys, list)
or not right_keys
or any(not _non_empty_text(item) for item in right_keys)
):
diagnostics.append(
_node_field_error(
node,
"reconcile.right_keys",
"Add at least one observed-table key.",
"right_keys",
)
)
if (
isinstance(left_keys, list)
and isinstance(right_keys, list)
and len(left_keys) != len(right_keys)
):
diagnostics.append(
_node_field_error(
node,
"reconcile.key_count",
"Expected and observed keys must have the same length.",
"right_keys",
)
)
prefix = config.get("right_prefix", "observed_")
if (
not _non_empty_text(prefix)
or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*_", str(prefix)) is None
):
diagnostics.append(
_node_field_error(
node,
"reconcile.right_prefix",
"Enter an identifier prefix ending in an underscore.",
"right_prefix",
)
)
elif node.type == "subflow":
if not _non_empty_text(config.get("template_ref")):
diagnostics.append(
_node_field_error(
node,
"subflow.template_ref",
"Choose a reusable template.",
"template_ref",
)
)
if not _non_empty_text(config.get("template_version")):
diagnostics.append(
_node_field_error(
node,
"subflow.template_version",
"Pin a template version.",
"template_version",
)
)
parameters = config.get("parameters")
if not isinstance(parameters, dict) or len(parameters) > 100:
diagnostics.append(
_node_field_error(
node,
"subflow.parameters",
"Subflow parameters must be an object with at most 100 entries.",
"parameters",
)
)
try:
nested_payload = substitute_parameters(
config.get("graph"),
parameters if isinstance(parameters, dict) else {},
)
nested = PipelineGraph.model_validate(nested_payload)
except (SubflowParameterError, ValueError) as exc:
diagnostics.append(
_node_field_error(
node,
"subflow.graph",
f"Pinned subflow graph is invalid: {exc}",
"graph",
)
)
else:
bound_inputs = [
item
for item in nested.nodes
if item.type == "source.inline"
and item.config.get("input_binding") is True
]
if len(bound_inputs) != 1:
diagnostics.append(
_node_field_error(
node,
"subflow.input_binding",
"Pinned subflows need exactly one inline input binding.",
"graph",
)
)
for nested_diagnostic in validate_graph(nested):
if nested_diagnostic.severity == "error":
diagnostics.append(
_node_field_error(
node,
"subflow.graph_validation",
f"Pinned subflow: {nested_diagnostic.message}",
"graph",
)
)
break
return diagnostics
@@ -940,6 +1617,18 @@ def _warning(
)
def _register_config_validators() -> None:
for node_type in SUPPORTED_NODE_TYPES:
if OPERATOR_REGISTRY.config_validator(node_type) is None:
OPERATOR_REGISTRY.register_config_validator(
node_type,
_validate_node_config,
)
_register_config_validators()
__all__ = [
"AGGREGATE_FUNCTIONS",
"DERIVE_OPERATIONS",