626 lines
21 KiB
Python
626 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Literal
|
|
|
|
from govoplan_core.core.definition_graphs import (
|
|
DefinitionConfigField,
|
|
DefinitionGraphConstraints,
|
|
DefinitionGraphLibrary,
|
|
DefinitionNodeCountConstraint,
|
|
DefinitionNodeType,
|
|
DefinitionPort,
|
|
)
|
|
|
|
|
|
NodeCategory = Literal[
|
|
"load",
|
|
"combine",
|
|
"filter",
|
|
"transform",
|
|
"quality",
|
|
"output",
|
|
]
|
|
SqlSupport = Literal["full", "partial", "none"]
|
|
NodePortDefinition = DefinitionPort
|
|
NodeConfigField = DefinitionConfigField
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class NodeTypeDefinition(DefinitionNodeType):
|
|
sql_support: SqlSupport = "full"
|
|
|
|
|
|
_NODE_TYPES = (
|
|
NodeTypeDefinition(
|
|
type="source.inline",
|
|
category="load",
|
|
label="Inline data",
|
|
description="Enter a small JSON table directly in the pipeline.",
|
|
icon="braces",
|
|
config_fields=(
|
|
NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True),
|
|
NodeConfigField(id="rows", label="Rows", kind="json_rows", required=True),
|
|
),
|
|
default_config={"source_name": "inline_source", "rows": []},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="source.reference",
|
|
category="load",
|
|
label="Datasource",
|
|
description="Load a bounded, fingerprinted state from the Datasources catalogue.",
|
|
icon="database",
|
|
config_fields=(
|
|
NodeConfigField(id="source_ref", label="Datasource", kind="datasource", required=True),
|
|
NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True),
|
|
NodeConfigField(id="expected_fingerprint", label="Expected fingerprint", kind="readonly"),
|
|
NodeConfigField(
|
|
id="consistency",
|
|
label="State",
|
|
kind="select",
|
|
options=(
|
|
("current", "Current"),
|
|
("live", "Live"),
|
|
("frozen", "Latest frozen"),
|
|
),
|
|
),
|
|
),
|
|
default_config={
|
|
"source_ref": "",
|
|
"source_name": "datasource",
|
|
"expected_fingerprint": "",
|
|
"consistency": "current",
|
|
},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="combine.union",
|
|
category="combine",
|
|
label="Append rows",
|
|
description="Append two or more inputs by column name.",
|
|
icon="combine",
|
|
input_ports=(
|
|
NodePortDefinition(
|
|
id="input",
|
|
label="Inputs",
|
|
multiple=True,
|
|
minimum_connections=2,
|
|
),
|
|
),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="mode",
|
|
label="Duplicates",
|
|
kind="select",
|
|
options=(("all", "Keep all"), ("distinct", "Remove duplicates")),
|
|
),
|
|
),
|
|
default_config={"mode": "all"},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="combine.join",
|
|
category="combine",
|
|
label="Join tables",
|
|
description="Match two inputs using one or more key columns.",
|
|
icon="git-merge",
|
|
input_ports=(
|
|
NodePortDefinition(id="left", label="Left"),
|
|
NodePortDefinition(id="right", label="Right"),
|
|
),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="join_type",
|
|
label="Join type",
|
|
kind="select",
|
|
options=(
|
|
("inner", "Matching rows"),
|
|
("left", "All left rows"),
|
|
("right", "All right rows"),
|
|
("full", "All rows"),
|
|
("semi", "Left rows with a match"),
|
|
("anti", "Left rows without a match"),
|
|
),
|
|
),
|
|
NodeConfigField(id="left_keys", label="Left keys", kind="column_list", required=True),
|
|
NodeConfigField(id="right_keys", label="Right keys", kind="column_list", required=True),
|
|
NodeConfigField(id="right_prefix", label="Right-column prefix", kind="text", required=True),
|
|
),
|
|
default_config={
|
|
"join_type": "inner",
|
|
"left_keys": [""],
|
|
"right_keys": [""],
|
|
"right_prefix": "right_",
|
|
},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="filter",
|
|
category="filter",
|
|
label="Filter rows",
|
|
description="Keep rows that satisfy a comparison.",
|
|
icon="filter",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="column", label="Column", kind="column", required=True),
|
|
NodeConfigField(
|
|
id="operator",
|
|
label="Operator",
|
|
kind="select",
|
|
required=True,
|
|
options=(
|
|
("eq", "Equals"),
|
|
("ne", "Does not equal"),
|
|
("gt", "Greater than"),
|
|
("gte", "Greater than or equal"),
|
|
("lt", "Less than"),
|
|
("lte", "Less than or equal"),
|
|
("contains", "Contains"),
|
|
("is_null", "Is empty"),
|
|
("not_null", "Is not empty"),
|
|
),
|
|
),
|
|
NodeConfigField(id="value", label="Value", kind="value"),
|
|
),
|
|
default_config={"column": "", "operator": "eq", "value": ""},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="filter.expression",
|
|
category="filter",
|
|
label="Expression filter",
|
|
description="Keep rows for which a safe typed expression is true.",
|
|
icon="list-checks",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="expression",
|
|
label="Expression",
|
|
kind="expression",
|
|
required=True,
|
|
),
|
|
),
|
|
default_config={"expression": "true"},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="distinct",
|
|
category="filter",
|
|
label="Remove duplicates",
|
|
description="Keep the first row for each selected key.",
|
|
icon="list-filter",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="columns",
|
|
label="Key columns",
|
|
kind="column_list",
|
|
description="Leave empty to compare complete rows.",
|
|
),
|
|
),
|
|
default_config={"columns": []},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="select",
|
|
category="transform",
|
|
label="Select columns",
|
|
description="Choose, order, and rename output columns.",
|
|
icon="columns-3",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="fields", label="Fields", kind="field_mapping", required=True),
|
|
),
|
|
default_config={"fields": [{"column": "", "alias": ""}]},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="derive",
|
|
category="transform",
|
|
label="Derive column",
|
|
description="Create a column through a constrained reusable operation.",
|
|
icon="variable",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="target_column", label="Output column", kind="text", required=True),
|
|
NodeConfigField(
|
|
id="operation",
|
|
label="Operation",
|
|
kind="select",
|
|
required=True,
|
|
options=(
|
|
("copy", "Copy"),
|
|
("upper", "Uppercase"),
|
|
("lower", "Lowercase"),
|
|
("trim", "Trim whitespace"),
|
|
("concat", "Concatenate"),
|
|
("coalesce", "First non-empty"),
|
|
("add", "Add"),
|
|
("subtract", "Subtract"),
|
|
("multiply", "Multiply"),
|
|
("divide", "Divide"),
|
|
),
|
|
),
|
|
NodeConfigField(id="source_columns", label="Source columns", kind="column_list", required=True),
|
|
NodeConfigField(id="separator", label="Separator", kind="text"),
|
|
),
|
|
default_config={
|
|
"target_column": "",
|
|
"operation": "copy",
|
|
"source_columns": [""],
|
|
"separator": " ",
|
|
},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="expression",
|
|
category="transform",
|
|
label="Expression",
|
|
description="Create or replace a column using a safe typed expression.",
|
|
icon="function-square",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="target_column",
|
|
label="Output column",
|
|
kind="text",
|
|
required=True,
|
|
),
|
|
NodeConfigField(
|
|
id="expression",
|
|
label="Expression",
|
|
kind="expression",
|
|
required=True,
|
|
),
|
|
NodeConfigField(
|
|
id="result_type",
|
|
label="Expected type",
|
|
kind="select",
|
|
options=(
|
|
("unknown", "Infer"),
|
|
("string", "Text"),
|
|
("integer", "Integer"),
|
|
("number", "Number"),
|
|
("boolean", "Boolean"),
|
|
("date", "Date"),
|
|
("datetime", "Date and time"),
|
|
),
|
|
),
|
|
),
|
|
default_config={
|
|
"target_column": "",
|
|
"expression": "",
|
|
"result_type": "unknown",
|
|
},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="calculate",
|
|
category="transform",
|
|
label="Calculate columns",
|
|
description=(
|
|
"Create or replace several columns using ordered safe typed "
|
|
"expressions."
|
|
),
|
|
icon="calculator",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="calculations",
|
|
label="Calculations",
|
|
kind="calculations",
|
|
required=True,
|
|
),
|
|
),
|
|
default_config={
|
|
"calculations": [
|
|
{
|
|
"target_column": "",
|
|
"expression": "",
|
|
"result_type": "unknown",
|
|
}
|
|
]
|
|
},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="convert",
|
|
category="transform",
|
|
label="Convert type",
|
|
description="Convert a column to a declared data type with explicit error handling.",
|
|
icon="replace-all",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="source_column", label="Source column", kind="column", required=True),
|
|
NodeConfigField(id="target_column", label="Output column", kind="text", required=True),
|
|
NodeConfigField(
|
|
id="target_type",
|
|
label="Data type",
|
|
kind="select",
|
|
required=True,
|
|
options=(
|
|
("string", "Text"),
|
|
("integer", "Integer"),
|
|
("number", "Number"),
|
|
("boolean", "Boolean"),
|
|
("date", "Date"),
|
|
("datetime", "Date and time"),
|
|
),
|
|
),
|
|
NodeConfigField(
|
|
id="on_error",
|
|
label="Conversion error",
|
|
kind="select",
|
|
options=(
|
|
("fail", "Stop"),
|
|
("null", "Use null"),
|
|
("keep", "Keep original"),
|
|
),
|
|
),
|
|
),
|
|
default_config={
|
|
"source_column": "",
|
|
"target_column": "",
|
|
"target_type": "string",
|
|
"on_error": "fail",
|
|
},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="replace",
|
|
category="transform",
|
|
label="Replace values",
|
|
description="Replace exact values or text fragments in a column.",
|
|
icon="replace",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="source_column", label="Source column", kind="column", required=True),
|
|
NodeConfigField(id="target_column", label="Output column", kind="text", required=True),
|
|
NodeConfigField(
|
|
id="mode",
|
|
label="Mode",
|
|
kind="select",
|
|
options=(("exact", "Exact value"), ("text", "Text fragment")),
|
|
),
|
|
NodeConfigField(id="find", label="Find", kind="value"),
|
|
NodeConfigField(id="replacement", label="Replacement", kind="value"),
|
|
),
|
|
default_config={
|
|
"source_column": "",
|
|
"target_column": "",
|
|
"mode": "exact",
|
|
"find": "",
|
|
"replacement": "",
|
|
},
|
|
sql_support="none",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="aggregate",
|
|
category="transform",
|
|
label="Aggregate",
|
|
description="Group rows and calculate counts or numeric summaries.",
|
|
icon="sigma",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="group_by", label="Group by", kind="column_list"),
|
|
NodeConfigField(id="aggregates", label="Calculations", kind="aggregates", required=True),
|
|
),
|
|
default_config={
|
|
"group_by": [],
|
|
"aggregates": [{"function": "count", "column": "*", "alias": "row_count"}],
|
|
},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="sort",
|
|
category="transform",
|
|
label="Sort rows",
|
|
description="Order rows by one or more columns.",
|
|
icon="arrow-up-down",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(NodeConfigField(id="fields", label="Sort fields", kind="sort_fields", required=True),),
|
|
default_config={"fields": [{"column": "", "direction": "asc"}]},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="window.rank",
|
|
category="transform",
|
|
label="Rank rows",
|
|
description=(
|
|
"Number or rank rows within optional partitions using a stable "
|
|
"ordering."
|
|
),
|
|
icon="list-ordered",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(
|
|
id="method",
|
|
label="Method",
|
|
kind="select",
|
|
options=(
|
|
("row_number", "Row number"),
|
|
("rank", "Rank with gaps"),
|
|
("dense_rank", "Dense rank"),
|
|
),
|
|
),
|
|
NodeConfigField(
|
|
id="target_column",
|
|
label="Output column",
|
|
kind="text",
|
|
required=True,
|
|
),
|
|
NodeConfigField(
|
|
id="partition_by",
|
|
label="Partition by",
|
|
kind="column_list",
|
|
),
|
|
NodeConfigField(
|
|
id="order_by",
|
|
label="Order by",
|
|
kind="sort_fields",
|
|
required=True,
|
|
),
|
|
),
|
|
default_config={
|
|
"method": "row_number",
|
|
"target_column": "row_number",
|
|
"partition_by": [],
|
|
"order_by": [{"column": "", "direction": "asc"}],
|
|
},
|
|
sql_support="partial",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="limit",
|
|
category="transform",
|
|
label="Limit rows",
|
|
description="Keep only the first number of rows.",
|
|
icon="list-end",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(NodeConfigField(id="count", label="Rows", kind="number", required=True),),
|
|
default_config={"count": 100},
|
|
),
|
|
NodeTypeDefinition(
|
|
type="quality.rules",
|
|
category="quality",
|
|
label="Quality rules",
|
|
description="Validate rows and annotate, drop, or reject invalid data.",
|
|
icon="badge-check",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="rules", label="Rules", kind="quality_rules", required=True),
|
|
NodeConfigField(
|
|
id="action",
|
|
label="Invalid rows",
|
|
kind="select",
|
|
options=(
|
|
("annotate", "Annotate"),
|
|
("drop", "Drop"),
|
|
("fail", "Stop"),
|
|
),
|
|
),
|
|
),
|
|
default_config={
|
|
"rules": [
|
|
{
|
|
"id": "required",
|
|
"column": "",
|
|
"operator": "not_null",
|
|
}
|
|
],
|
|
"action": "annotate",
|
|
},
|
|
sql_support="none",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="reconcile.compare",
|
|
category="quality",
|
|
label="Reconcile tables",
|
|
description="Compare keyed rows and expose missing, matching, and changed records.",
|
|
icon="scan-search",
|
|
input_ports=(
|
|
NodePortDefinition(id="left", label="Expected"),
|
|
NodePortDefinition(id="right", label="Observed"),
|
|
),
|
|
config_fields=(
|
|
NodeConfigField(id="left_keys", label="Expected keys", kind="column_list", required=True),
|
|
NodeConfigField(id="right_keys", label="Observed keys", kind="column_list", required=True),
|
|
NodeConfigField(id="compare_columns", label="Compared columns", kind="field_mapping"),
|
|
NodeConfigField(id="right_prefix", label="Observed prefix", kind="text", required=True),
|
|
),
|
|
default_config={
|
|
"left_keys": [""],
|
|
"right_keys": [""],
|
|
"compare_columns": [],
|
|
"right_prefix": "observed_",
|
|
},
|
|
sql_support="none",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="subflow",
|
|
category="transform",
|
|
label="Reusable subflow",
|
|
description="Run a pinned parameterized template snapshot as one node.",
|
|
icon="boxes",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
config_fields=(
|
|
NodeConfigField(id="template_ref", label="Template reference", kind="text", required=True),
|
|
NodeConfigField(id="template_version", label="Template version", kind="text", required=True),
|
|
NodeConfigField(id="parameters", label="Parameters", kind="json", required=True),
|
|
NodeConfigField(id="graph", label="Pinned graph", kind="json", required=True),
|
|
),
|
|
default_config={
|
|
"template_ref": "",
|
|
"template_version": "",
|
|
"parameters": {},
|
|
"graph": {
|
|
"schema_version": 1,
|
|
"nodes": [],
|
|
"edges": [],
|
|
},
|
|
},
|
|
sql_support="none",
|
|
),
|
|
NodeTypeDefinition(
|
|
type="output",
|
|
category="output",
|
|
label="Preview output",
|
|
description="Expose the terminal table for preview or publication.",
|
|
icon="panel-top-open",
|
|
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
|
output_ports=(),
|
|
default_config={},
|
|
),
|
|
)
|
|
|
|
CATEGORY_LABELS = {
|
|
"load": "Load",
|
|
"combine": "Combine",
|
|
"filter": "Filter",
|
|
"transform": "Transform",
|
|
"quality": "Quality",
|
|
"output": "Output",
|
|
}
|
|
DATAFLOW_GRAPH_LIBRARY = DefinitionGraphLibrary(
|
|
id="dataflow",
|
|
version="0.1.0",
|
|
category_labels=CATEGORY_LABELS,
|
|
node_types=_NODE_TYPES,
|
|
constraints=DefinitionGraphConstraints(
|
|
max_nodes=100,
|
|
max_edges=200,
|
|
allow_cycles=False,
|
|
require_connected=True,
|
|
node_counts=(
|
|
DefinitionNodeCountConstraint(
|
|
code="graph.source_count",
|
|
label="source",
|
|
minimum=1,
|
|
maximum=10,
|
|
type_prefixes=("source.",),
|
|
),
|
|
DefinitionNodeCountConstraint(
|
|
code="graph.output_count",
|
|
label="output",
|
|
minimum=1,
|
|
maximum=1,
|
|
node_types=("output",),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
NODE_LIBRARY: tuple[NodeTypeDefinition, ...] = _NODE_TYPES
|
|
NODE_TYPES = {definition.type: definition for definition in NODE_LIBRARY}
|
|
|
|
|
|
def node_definition(node_type: str) -> NodeTypeDefinition | None:
|
|
return NODE_TYPES.get(node_type)
|
|
|
|
|
|
__all__ = [
|
|
"CATEGORY_LABELS",
|
|
"DATAFLOW_GRAPH_LIBRARY",
|
|
"NODE_LIBRARY",
|
|
"NODE_TYPES",
|
|
"NodeCategory",
|
|
"NodeConfigField",
|
|
"NodePortDefinition",
|
|
"NodeTypeDefinition",
|
|
"SqlSupport",
|
|
"node_definition",
|
|
]
|