from __future__ import annotations from dataclasses import dataclass from typing import Literal from govoplan_core.core.definition_graphs import ( DefinitionConfigField, DefinitionGraphConstraints, DefinitionGraphLibrary, DefinitionNodeCountConstraint, DefinitionNodeType, DefinitionPort, ) NodeCategory = Literal[ "load", "combine", "filter", "transform", "quality", "output", ] SqlSupport = Literal["full", "partial", "none"] NodePortDefinition = DefinitionPort NodeConfigField = DefinitionConfigField @dataclass(frozen=True, slots=True) class NodeTypeDefinition(DefinitionNodeType): sql_support: SqlSupport = "full" _NODE_TYPES = ( NodeTypeDefinition( type="source.inline", category="load", label="Inline data", description="Enter a small JSON table directly in the pipeline.", icon="braces", config_fields=( NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True), NodeConfigField(id="rows", label="Rows", kind="json_rows", required=True), ), default_config={"source_name": "inline_source", "rows": []}, ), NodeTypeDefinition( type="source.reference", category="load", label="Datasource", description="Load a bounded, fingerprinted state from the Datasources catalogue.", icon="database", config_fields=( NodeConfigField(id="source_ref", label="Datasource", kind="datasource", required=True), NodeConfigField(id="source_name", label="SQL source name", kind="text", required=True), NodeConfigField(id="expected_fingerprint", label="Expected fingerprint", kind="readonly"), NodeConfigField( id="consistency", label="State", kind="select", options=( ("current", "Current"), ("live", "Live"), ("frozen", "Latest frozen"), ), ), ), default_config={ "source_ref": "", "source_name": "datasource", "expected_fingerprint": "", "consistency": "current", }, ), NodeTypeDefinition( type="combine.union", category="combine", label="Append rows", description="Append two or more inputs by column name.", icon="combine", input_ports=( NodePortDefinition( id="input", label="Inputs", multiple=True, minimum_connections=2, ), ), config_fields=( NodeConfigField( id="mode", label="Duplicates", kind="select", options=(("all", "Keep all"), ("distinct", "Remove duplicates")), ), ), default_config={"mode": "all"}, sql_support="partial", ), NodeTypeDefinition( type="combine.join", category="combine", label="Join tables", description="Match two inputs using one or more key columns.", icon="git-merge", input_ports=( NodePortDefinition(id="left", label="Left"), NodePortDefinition(id="right", label="Right"), ), config_fields=( NodeConfigField( id="join_type", label="Join type", kind="select", options=( ("inner", "Matching rows"), ("left", "All left rows"), ("right", "All right rows"), ("full", "All rows"), ("semi", "Left rows with a match"), ("anti", "Left rows without a match"), ), ), NodeConfigField(id="left_keys", label="Left keys", kind="column_list", required=True), NodeConfigField(id="right_keys", label="Right keys", kind="column_list", required=True), NodeConfigField(id="right_prefix", label="Right-column prefix", kind="text", required=True), ), default_config={ "join_type": "inner", "left_keys": [""], "right_keys": [""], "right_prefix": "right_", }, sql_support="partial", ), NodeTypeDefinition( type="filter", category="filter", label="Filter rows", description="Keep rows that satisfy a comparison.", icon="filter", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="column", label="Column", kind="column", required=True), NodeConfigField( id="operator", label="Operator", kind="select", required=True, options=( ("eq", "Equals"), ("ne", "Does not equal"), ("gt", "Greater than"), ("gte", "Greater than or equal"), ("lt", "Less than"), ("lte", "Less than or equal"), ("contains", "Contains"), ("is_null", "Is empty"), ("not_null", "Is not empty"), ), ), NodeConfigField(id="value", label="Value", kind="value"), ), default_config={"column": "", "operator": "eq", "value": ""}, ), NodeTypeDefinition( type="filter.expression", category="filter", label="Expression filter", description="Keep rows for which a safe typed expression is true.", icon="list-checks", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField( id="expression", label="Expression", kind="expression", required=True, ), ), default_config={"expression": "true"}, sql_support="partial", ), NodeTypeDefinition( type="distinct", category="filter", label="Remove duplicates", description="Keep the first row for each selected key.", icon="list-filter", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField( id="columns", label="Key columns", kind="column_list", description="Leave empty to compare complete rows.", ), ), default_config={"columns": []}, sql_support="partial", ), NodeTypeDefinition( type="select", category="transform", label="Select columns", description="Choose, order, and rename output columns.", icon="columns-3", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="fields", label="Fields", kind="field_mapping", required=True), ), default_config={"fields": [{"column": "", "alias": ""}]}, ), NodeTypeDefinition( type="derive", category="transform", label="Derive column", description="Create a column through a constrained reusable operation.", icon="variable", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="target_column", label="Output column", kind="text", required=True), NodeConfigField( id="operation", label="Operation", kind="select", required=True, options=( ("copy", "Copy"), ("upper", "Uppercase"), ("lower", "Lowercase"), ("trim", "Trim whitespace"), ("concat", "Concatenate"), ("coalesce", "First non-empty"), ("add", "Add"), ("subtract", "Subtract"), ("multiply", "Multiply"), ("divide", "Divide"), ), ), NodeConfigField(id="source_columns", label="Source columns", kind="column_list", required=True), NodeConfigField(id="separator", label="Separator", kind="text"), ), default_config={ "target_column": "", "operation": "copy", "source_columns": [""], "separator": " ", }, sql_support="partial", ), NodeTypeDefinition( type="expression", category="transform", label="Expression", description="Create or replace a column using a safe typed expression.", icon="function-square", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField( id="target_column", label="Output column", kind="text", required=True, ), NodeConfigField( id="expression", label="Expression", kind="expression", required=True, ), NodeConfigField( id="result_type", label="Expected type", kind="select", options=( ("unknown", "Infer"), ("string", "Text"), ("integer", "Integer"), ("number", "Number"), ("boolean", "Boolean"), ("date", "Date"), ("datetime", "Date and time"), ), ), ), default_config={ "target_column": "", "expression": "", "result_type": "unknown", }, sql_support="partial", ), NodeTypeDefinition( type="calculate", category="transform", label="Calculate columns", description=( "Create or replace several columns using ordered safe typed " "expressions." ), icon="calculator", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField( id="calculations", label="Calculations", kind="calculations", required=True, ), ), default_config={ "calculations": [ { "target_column": "", "expression": "", "result_type": "unknown", } ] }, sql_support="partial", ), NodeTypeDefinition( type="convert", category="transform", label="Convert type", description="Convert a column to a declared data type with explicit error handling.", icon="replace-all", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="source_column", label="Source column", kind="column", required=True), NodeConfigField(id="target_column", label="Output column", kind="text", required=True), NodeConfigField( id="target_type", label="Data type", kind="select", required=True, options=( ("string", "Text"), ("integer", "Integer"), ("number", "Number"), ("boolean", "Boolean"), ("date", "Date"), ("datetime", "Date and time"), ), ), NodeConfigField( id="on_error", label="Conversion error", kind="select", options=( ("fail", "Stop"), ("null", "Use null"), ("keep", "Keep original"), ), ), ), default_config={ "source_column": "", "target_column": "", "target_type": "string", "on_error": "fail", }, sql_support="partial", ), NodeTypeDefinition( type="replace", category="transform", label="Replace values", description="Replace exact values or text fragments in a column.", icon="replace", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="source_column", label="Source column", kind="column", required=True), NodeConfigField(id="target_column", label="Output column", kind="text", required=True), NodeConfigField( id="mode", label="Mode", kind="select", options=(("exact", "Exact value"), ("text", "Text fragment")), ), NodeConfigField(id="find", label="Find", kind="value"), NodeConfigField(id="replacement", label="Replacement", kind="value"), ), default_config={ "source_column": "", "target_column": "", "mode": "exact", "find": "", "replacement": "", }, sql_support="none", ), NodeTypeDefinition( type="aggregate", category="transform", label="Aggregate", description="Group rows and calculate counts or numeric summaries.", icon="sigma", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="group_by", label="Group by", kind="column_list"), NodeConfigField(id="aggregates", label="Calculations", kind="aggregates", required=True), ), default_config={ "group_by": [], "aggregates": [{"function": "count", "column": "*", "alias": "row_count"}], }, ), NodeTypeDefinition( type="sort", category="transform", label="Sort rows", description="Order rows by one or more columns.", icon="arrow-up-down", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=(NodeConfigField(id="fields", label="Sort fields", kind="sort_fields", required=True),), default_config={"fields": [{"column": "", "direction": "asc"}]}, ), NodeTypeDefinition( type="window.rank", category="transform", label="Rank rows", description=( "Number or rank rows within optional partitions using a stable " "ordering." ), icon="list-ordered", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField( id="method", label="Method", kind="select", options=( ("row_number", "Row number"), ("rank", "Rank with gaps"), ("dense_rank", "Dense rank"), ), ), NodeConfigField( id="target_column", label="Output column", kind="text", required=True, ), NodeConfigField( id="partition_by", label="Partition by", kind="column_list", ), NodeConfigField( id="order_by", label="Order by", kind="sort_fields", required=True, ), ), default_config={ "method": "row_number", "target_column": "row_number", "partition_by": [], "order_by": [{"column": "", "direction": "asc"}], }, sql_support="partial", ), NodeTypeDefinition( type="limit", category="transform", label="Limit rows", description="Keep only the first number of rows.", icon="list-end", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=(NodeConfigField(id="count", label="Rows", kind="number", required=True),), default_config={"count": 100}, ), NodeTypeDefinition( type="quality.rules", category="quality", label="Quality rules", description="Validate rows and annotate, drop, or reject invalid data.", icon="badge-check", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="rules", label="Rules", kind="quality_rules", required=True), NodeConfigField( id="action", label="Invalid rows", kind="select", options=( ("annotate", "Annotate"), ("drop", "Drop"), ("fail", "Stop"), ), ), ), default_config={ "rules": [ { "id": "required", "column": "", "operator": "not_null", } ], "action": "annotate", }, sql_support="none", ), NodeTypeDefinition( type="reconcile.compare", category="quality", label="Reconcile tables", description="Compare keyed rows and expose missing, matching, and changed records.", icon="scan-search", input_ports=( NodePortDefinition(id="left", label="Expected"), NodePortDefinition(id="right", label="Observed"), ), config_fields=( NodeConfigField(id="left_keys", label="Expected keys", kind="column_list", required=True), NodeConfigField(id="right_keys", label="Observed keys", kind="column_list", required=True), NodeConfigField(id="compare_columns", label="Compared columns", kind="field_mapping"), NodeConfigField(id="right_prefix", label="Observed prefix", kind="text", required=True), ), default_config={ "left_keys": [""], "right_keys": [""], "compare_columns": [], "right_prefix": "observed_", }, sql_support="none", ), NodeTypeDefinition( type="subflow", category="transform", label="Reusable subflow", description="Run a pinned parameterized template snapshot as one node.", icon="boxes", input_ports=(NodePortDefinition(id="input", label="Input"),), config_fields=( NodeConfigField(id="template_ref", label="Template reference", kind="text", required=True), NodeConfigField(id="template_version", label="Template version", kind="text", required=True), NodeConfigField(id="parameters", label="Parameters", kind="json", required=True), NodeConfigField(id="graph", label="Pinned graph", kind="json", required=True), ), default_config={ "template_ref": "", "template_version": "", "parameters": {}, "graph": { "schema_version": 1, "nodes": [], "edges": [], }, }, sql_support="none", ), NodeTypeDefinition( type="output", category="output", label="Preview output", description="Expose the terminal table for preview or publication.", icon="panel-top-open", input_ports=(NodePortDefinition(id="input", label="Input"),), output_ports=(), default_config={}, ), ) CATEGORY_LABELS = { "load": "Load", "combine": "Combine", "filter": "Filter", "transform": "Transform", "quality": "Quality", "output": "Output", } DATAFLOW_GRAPH_LIBRARY = DefinitionGraphLibrary( id="dataflow", version="0.1.0", category_labels=CATEGORY_LABELS, node_types=_NODE_TYPES, constraints=DefinitionGraphConstraints( max_nodes=100, max_edges=200, allow_cycles=False, require_connected=True, node_counts=( DefinitionNodeCountConstraint( code="graph.source_count", label="source", minimum=1, maximum=10, type_prefixes=("source.",), ), DefinitionNodeCountConstraint( code="graph.output_count", label="output", minimum=1, maximum=1, node_types=("output",), ), ), ), ) NODE_LIBRARY: tuple[NodeTypeDefinition, ...] = _NODE_TYPES NODE_TYPES = {definition.type: definition for definition in NODE_LIBRARY} def node_definition(node_type: str) -> NodeTypeDefinition | None: return NODE_TYPES.get(node_type) __all__ = [ "CATEGORY_LABELS", "DATAFLOW_GRAPH_LIBRARY", "NODE_LIBRARY", "NODE_TYPES", "NodeCategory", "NodeConfigField", "NodePortDefinition", "NodeTypeDefinition", "SqlSupport", "node_definition", ]