feat: add extensible operators and golden flows
This commit is contained in:
@@ -13,7 +13,14 @@ from govoplan_core.core.definition_graphs import (
|
||||
)
|
||||
|
||||
|
||||
NodeCategory = Literal["load", "combine", "filter", "transform", "output"]
|
||||
NodeCategory = Literal[
|
||||
"load",
|
||||
"combine",
|
||||
"filter",
|
||||
"transform",
|
||||
"quality",
|
||||
"output",
|
||||
]
|
||||
SqlSupport = Literal["full", "partial", "none"]
|
||||
NodePortDefinition = DefinitionPort
|
||||
NodeConfigField = DefinitionConfigField
|
||||
@@ -154,6 +161,24 @@ _NODE_TYPES = (
|
||||
),
|
||||
default_config={"column": "", "operator": "eq", "value": ""},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="filter.expression",
|
||||
category="filter",
|
||||
label="Expression filter",
|
||||
description="Keep rows for which a safe typed expression is true.",
|
||||
icon="list-checks",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(
|
||||
id="expression",
|
||||
label="Expression",
|
||||
kind="expression",
|
||||
required=True,
|
||||
),
|
||||
),
|
||||
default_config={"expression": "true"},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="distinct",
|
||||
category="filter",
|
||||
@@ -222,6 +247,119 @@ _NODE_TYPES = (
|
||||
},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="expression",
|
||||
category="transform",
|
||||
label="Expression",
|
||||
description="Create or replace a column using a safe typed expression.",
|
||||
icon="function-square",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(
|
||||
id="target_column",
|
||||
label="Output column",
|
||||
kind="text",
|
||||
required=True,
|
||||
),
|
||||
NodeConfigField(
|
||||
id="expression",
|
||||
label="Expression",
|
||||
kind="expression",
|
||||
required=True,
|
||||
),
|
||||
NodeConfigField(
|
||||
id="result_type",
|
||||
label="Expected type",
|
||||
kind="select",
|
||||
options=(
|
||||
("unknown", "Infer"),
|
||||
("string", "Text"),
|
||||
("integer", "Integer"),
|
||||
("number", "Number"),
|
||||
("boolean", "Boolean"),
|
||||
("date", "Date"),
|
||||
("datetime", "Date and time"),
|
||||
),
|
||||
),
|
||||
),
|
||||
default_config={
|
||||
"target_column": "",
|
||||
"expression": "",
|
||||
"result_type": "unknown",
|
||||
},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="convert",
|
||||
category="transform",
|
||||
label="Convert type",
|
||||
description="Convert a column to a declared data type with explicit error handling.",
|
||||
icon="replace-all",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="source_column", label="Source column", kind="column", required=True),
|
||||
NodeConfigField(id="target_column", label="Output column", kind="text", required=True),
|
||||
NodeConfigField(
|
||||
id="target_type",
|
||||
label="Data type",
|
||||
kind="select",
|
||||
required=True,
|
||||
options=(
|
||||
("string", "Text"),
|
||||
("integer", "Integer"),
|
||||
("number", "Number"),
|
||||
("boolean", "Boolean"),
|
||||
("date", "Date"),
|
||||
("datetime", "Date and time"),
|
||||
),
|
||||
),
|
||||
NodeConfigField(
|
||||
id="on_error",
|
||||
label="Conversion error",
|
||||
kind="select",
|
||||
options=(
|
||||
("fail", "Stop"),
|
||||
("null", "Use null"),
|
||||
("keep", "Keep original"),
|
||||
),
|
||||
),
|
||||
),
|
||||
default_config={
|
||||
"source_column": "",
|
||||
"target_column": "",
|
||||
"target_type": "string",
|
||||
"on_error": "fail",
|
||||
},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="replace",
|
||||
category="transform",
|
||||
label="Replace values",
|
||||
description="Replace exact values or text fragments in a column.",
|
||||
icon="replace",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="source_column", label="Source column", kind="column", required=True),
|
||||
NodeConfigField(id="target_column", label="Output column", kind="text", required=True),
|
||||
NodeConfigField(
|
||||
id="mode",
|
||||
label="Mode",
|
||||
kind="select",
|
||||
options=(("exact", "Exact value"), ("text", "Text fragment")),
|
||||
),
|
||||
NodeConfigField(id="find", label="Find", kind="value"),
|
||||
NodeConfigField(id="replacement", label="Replacement", kind="value"),
|
||||
),
|
||||
default_config={
|
||||
"source_column": "",
|
||||
"target_column": "",
|
||||
"mode": "exact",
|
||||
"find": "",
|
||||
"replacement": "",
|
||||
},
|
||||
sql_support="none",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="aggregate",
|
||||
category="transform",
|
||||
@@ -258,6 +396,87 @@ _NODE_TYPES = (
|
||||
config_fields=(NodeConfigField(id="count", label="Rows", kind="number", required=True),),
|
||||
default_config={"count": 100},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="quality.rules",
|
||||
category="quality",
|
||||
label="Quality rules",
|
||||
description="Validate rows and annotate, drop, or reject invalid data.",
|
||||
icon="badge-check",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="rules", label="Rules", kind="quality_rules", required=True),
|
||||
NodeConfigField(
|
||||
id="action",
|
||||
label="Invalid rows",
|
||||
kind="select",
|
||||
options=(
|
||||
("annotate", "Annotate"),
|
||||
("drop", "Drop"),
|
||||
("fail", "Stop"),
|
||||
),
|
||||
),
|
||||
),
|
||||
default_config={
|
||||
"rules": [
|
||||
{
|
||||
"id": "required",
|
||||
"column": "",
|
||||
"operator": "not_null",
|
||||
}
|
||||
],
|
||||
"action": "annotate",
|
||||
},
|
||||
sql_support="none",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="reconcile.compare",
|
||||
category="quality",
|
||||
label="Reconcile tables",
|
||||
description="Compare keyed rows and expose missing, matching, and changed records.",
|
||||
icon="scan-search",
|
||||
input_ports=(
|
||||
NodePortDefinition(id="left", label="Expected"),
|
||||
NodePortDefinition(id="right", label="Observed"),
|
||||
),
|
||||
config_fields=(
|
||||
NodeConfigField(id="left_keys", label="Expected keys", kind="column_list", required=True),
|
||||
NodeConfigField(id="right_keys", label="Observed keys", kind="column_list", required=True),
|
||||
NodeConfigField(id="compare_columns", label="Compared columns", kind="field_mapping"),
|
||||
NodeConfigField(id="right_prefix", label="Observed prefix", kind="text", required=True),
|
||||
),
|
||||
default_config={
|
||||
"left_keys": [""],
|
||||
"right_keys": [""],
|
||||
"compare_columns": [],
|
||||
"right_prefix": "observed_",
|
||||
},
|
||||
sql_support="none",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="subflow",
|
||||
category="transform",
|
||||
label="Reusable subflow",
|
||||
description="Run a pinned parameterized template snapshot as one node.",
|
||||
icon="boxes",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="template_ref", label="Template reference", kind="text", required=True),
|
||||
NodeConfigField(id="template_version", label="Template version", kind="text", required=True),
|
||||
NodeConfigField(id="parameters", label="Parameters", kind="json", required=True),
|
||||
NodeConfigField(id="graph", label="Pinned graph", kind="json", required=True),
|
||||
),
|
||||
default_config={
|
||||
"template_ref": "",
|
||||
"template_version": "",
|
||||
"parameters": {},
|
||||
"graph": {
|
||||
"schema_version": 1,
|
||||
"nodes": [],
|
||||
"edges": [],
|
||||
},
|
||||
},
|
||||
sql_support="none",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="output",
|
||||
category="output",
|
||||
@@ -275,6 +494,7 @@ CATEGORY_LABELS = {
|
||||
"combine": "Combine",
|
||||
"filter": "Filter",
|
||||
"transform": "Transform",
|
||||
"quality": "Quality",
|
||||
"output": "Output",
|
||||
}
|
||||
DATAFLOW_GRAPH_LIBRARY = DefinitionGraphLibrary(
|
||||
|
||||
Reference in New Issue
Block a user