Expand governed Dataflow editor and node library
This commit is contained in:
@@ -0,0 +1,305 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
NodeCategory = Literal["load", "combine", "filter", "transform", "output"]
|
||||
SqlSupport = Literal["full", "partial", "none"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NodePortDefinition:
|
||||
id: str
|
||||
label: str
|
||||
required: bool = True
|
||||
multiple: bool = False
|
||||
minimum_connections: int = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NodeConfigField:
|
||||
id: str
|
||||
label: str
|
||||
kind: str
|
||||
required: bool = False
|
||||
description: str | None = None
|
||||
options: tuple[tuple[str, str], ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NodeTypeDefinition:
|
||||
type: str
|
||||
category: NodeCategory
|
||||
label: str
|
||||
description: str
|
||||
icon: str
|
||||
input_ports: tuple[NodePortDefinition, ...] = ()
|
||||
output_ports: tuple[NodePortDefinition, ...] = (
|
||||
NodePortDefinition(id="output", label="Output"),
|
||||
)
|
||||
config_fields: tuple[NodeConfigField, ...] = ()
|
||||
default_config: dict[str, Any] = field(default_factory=dict)
|
||||
sql_support: SqlSupport = "full"
|
||||
|
||||
|
||||
NODE_LIBRARY = (
|
||||
NodeTypeDefinition(
|
||||
type="source.inline",
|
||||
category="load",
|
||||
label="Inline data",
|
||||
description="Enter a small JSON table directly in the pipeline.",
|
||||
icon="braces",
|
||||
config_fields=(
|
||||
NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True),
|
||||
NodeConfigField(id="rows", label="Rows", kind="json_rows", required=True),
|
||||
),
|
||||
default_config={"source_name": "inline_source", "rows": []},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="source.reference",
|
||||
category="load",
|
||||
label="Connector source",
|
||||
description="Load a bounded, fingerprinted table exposed by Connectors.",
|
||||
icon="database",
|
||||
config_fields=(
|
||||
NodeConfigField(id="source_ref", label="Source", kind="tabular_source", required=True),
|
||||
NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True),
|
||||
NodeConfigField(id="expected_fingerprint", label="Expected fingerprint", kind="readonly"),
|
||||
),
|
||||
default_config={
|
||||
"source_ref": "",
|
||||
"source_name": "connector_source",
|
||||
"expected_fingerprint": "",
|
||||
},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="combine.union",
|
||||
category="combine",
|
||||
label="Append rows",
|
||||
description="Append two or more inputs by column name.",
|
||||
icon="combine",
|
||||
input_ports=(
|
||||
NodePortDefinition(
|
||||
id="input",
|
||||
label="Inputs",
|
||||
multiple=True,
|
||||
minimum_connections=2,
|
||||
),
|
||||
),
|
||||
config_fields=(
|
||||
NodeConfigField(
|
||||
id="mode",
|
||||
label="Duplicates",
|
||||
kind="select",
|
||||
options=(("all", "Keep all"), ("distinct", "Remove duplicates")),
|
||||
),
|
||||
),
|
||||
default_config={"mode": "all"},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="combine.join",
|
||||
category="combine",
|
||||
label="Join tables",
|
||||
description="Match two inputs using one or more key columns.",
|
||||
icon="git-merge",
|
||||
input_ports=(
|
||||
NodePortDefinition(id="left", label="Left"),
|
||||
NodePortDefinition(id="right", label="Right"),
|
||||
),
|
||||
config_fields=(
|
||||
NodeConfigField(
|
||||
id="join_type",
|
||||
label="Join type",
|
||||
kind="select",
|
||||
options=(
|
||||
("inner", "Matching rows"),
|
||||
("left", "All left rows"),
|
||||
("right", "All right rows"),
|
||||
("full", "All rows"),
|
||||
),
|
||||
),
|
||||
NodeConfigField(id="left_keys", label="Left keys", kind="column_list", required=True),
|
||||
NodeConfigField(id="right_keys", label="Right keys", kind="column_list", required=True),
|
||||
NodeConfigField(id="right_prefix", label="Right-column prefix", kind="text", required=True),
|
||||
),
|
||||
default_config={
|
||||
"join_type": "inner",
|
||||
"left_keys": [""],
|
||||
"right_keys": [""],
|
||||
"right_prefix": "right_",
|
||||
},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="filter",
|
||||
category="filter",
|
||||
label="Filter rows",
|
||||
description="Keep rows that satisfy a comparison.",
|
||||
icon="filter",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="column", label="Column", kind="column", required=True),
|
||||
NodeConfigField(
|
||||
id="operator",
|
||||
label="Operator",
|
||||
kind="select",
|
||||
required=True,
|
||||
options=(
|
||||
("eq", "Equals"),
|
||||
("ne", "Does not equal"),
|
||||
("gt", "Greater than"),
|
||||
("gte", "Greater than or equal"),
|
||||
("lt", "Less than"),
|
||||
("lte", "Less than or equal"),
|
||||
("contains", "Contains"),
|
||||
("is_null", "Is empty"),
|
||||
("not_null", "Is not empty"),
|
||||
),
|
||||
),
|
||||
NodeConfigField(id="value", label="Value", kind="value"),
|
||||
),
|
||||
default_config={"column": "", "operator": "eq", "value": ""},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="distinct",
|
||||
category="filter",
|
||||
label="Remove duplicates",
|
||||
description="Keep the first row for each selected key.",
|
||||
icon="list-filter",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(
|
||||
id="columns",
|
||||
label="Key columns",
|
||||
kind="column_list",
|
||||
description="Leave empty to compare complete rows.",
|
||||
),
|
||||
),
|
||||
default_config={"columns": []},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="select",
|
||||
category="transform",
|
||||
label="Select columns",
|
||||
description="Choose, order, and rename output columns.",
|
||||
icon="columns-3",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="fields", label="Fields", kind="field_mapping", required=True),
|
||||
),
|
||||
default_config={"fields": [{"column": "", "alias": ""}]},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="derive",
|
||||
category="transform",
|
||||
label="Derive column",
|
||||
description="Create a column through a constrained reusable operation.",
|
||||
icon="variable",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="target_column", label="Output column", kind="text", required=True),
|
||||
NodeConfigField(
|
||||
id="operation",
|
||||
label="Operation",
|
||||
kind="select",
|
||||
required=True,
|
||||
options=(
|
||||
("copy", "Copy"),
|
||||
("upper", "Uppercase"),
|
||||
("lower", "Lowercase"),
|
||||
("trim", "Trim whitespace"),
|
||||
("concat", "Concatenate"),
|
||||
("coalesce", "First non-empty"),
|
||||
("add", "Add"),
|
||||
("subtract", "Subtract"),
|
||||
("multiply", "Multiply"),
|
||||
("divide", "Divide"),
|
||||
),
|
||||
),
|
||||
NodeConfigField(id="source_columns", label="Source columns", kind="column_list", required=True),
|
||||
NodeConfigField(id="separator", label="Separator", kind="text"),
|
||||
),
|
||||
default_config={
|
||||
"target_column": "",
|
||||
"operation": "copy",
|
||||
"source_columns": [""],
|
||||
"separator": " ",
|
||||
},
|
||||
sql_support="partial",
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="aggregate",
|
||||
category="transform",
|
||||
label="Aggregate",
|
||||
description="Group rows and calculate counts or numeric summaries.",
|
||||
icon="sigma",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(
|
||||
NodeConfigField(id="group_by", label="Group by", kind="column_list"),
|
||||
NodeConfigField(id="aggregates", label="Calculations", kind="aggregates", required=True),
|
||||
),
|
||||
default_config={
|
||||
"group_by": [],
|
||||
"aggregates": [{"function": "count", "column": "*", "alias": "row_count"}],
|
||||
},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="sort",
|
||||
category="transform",
|
||||
label="Sort rows",
|
||||
description="Order rows by one or more columns.",
|
||||
icon="arrow-up-down",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(NodeConfigField(id="fields", label="Sort fields", kind="sort_fields", required=True),),
|
||||
default_config={"fields": [{"column": "", "direction": "asc"}]},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="limit",
|
||||
category="transform",
|
||||
label="Limit rows",
|
||||
description="Keep only the first number of rows.",
|
||||
icon="list-end",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
config_fields=(NodeConfigField(id="count", label="Rows", kind="number", required=True),),
|
||||
default_config={"count": 100},
|
||||
),
|
||||
NodeTypeDefinition(
|
||||
type="output",
|
||||
category="output",
|
||||
label="Preview output",
|
||||
description="Expose the terminal table for preview or publication.",
|
||||
icon="panel-top-open",
|
||||
input_ports=(NodePortDefinition(id="input", label="Input"),),
|
||||
output_ports=(),
|
||||
default_config={},
|
||||
),
|
||||
)
|
||||
|
||||
NODE_TYPES = {definition.type: definition for definition in NODE_LIBRARY}
|
||||
CATEGORY_LABELS = {
|
||||
"load": "Load",
|
||||
"combine": "Combine",
|
||||
"filter": "Filter",
|
||||
"transform": "Transform",
|
||||
"output": "Output",
|
||||
}
|
||||
|
||||
|
||||
def node_definition(node_type: str) -> NodeTypeDefinition | None:
|
||||
return NODE_TYPES.get(node_type)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CATEGORY_LABELS",
|
||||
"NODE_LIBRARY",
|
||||
"NODE_TYPES",
|
||||
"NodeCategory",
|
||||
"NodeConfigField",
|
||||
"NodePortDefinition",
|
||||
"NodeTypeDefinition",
|
||||
"SqlSupport",
|
||||
"node_definition",
|
||||
]
|
||||
Reference in New Issue
Block a user