GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Initialize governed Dataflow module

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-28 01:28:17 +02:00
commit 223c008ff3
44 changed files with 6204 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/govoplan_dataflow/backend/__init__.py Normal file
View File

@@ -0,0 +1 @@
from __future__ import annotations

5
src/govoplan_dataflow/backend/db/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
from __future__ import annotations
from govoplan_dataflow.backend.db.models import DataflowPipeline, DataflowPipelineRevision, DataflowRun
__all__ = ["DataflowPipeline", "DataflowPipelineRevision", "DataflowRun"]

114
src/govoplan_dataflow/backend/db/models.py Normal file
View File

@@ -0,0 +1,114 @@
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any
from sqlalchemy import DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from govoplan_core.db.base import Base, TimestampMixin
def new_uuid() -> str:
return str(uuid.uuid4())
class DataflowPipeline(Base, TimestampMixin):
__tablename__ = "dataflow_pipelines"
__table_args__ = (
Index("ix_dataflow_pipelines_tenant_status", "tenant_id", "status"),
Index("ix_dataflow_pipelines_tenant_updated", "tenant_id", "updated_at"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(300), nullable=False)
description: Mapped[str | None] = mapped_column(Text)
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
current_revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict, nullable=False)
revisions: Mapped[list["DataflowPipelineRevision"]] = relationship(
back_populates="pipeline",
cascade="all, delete-orphan",
order_by="DataflowPipelineRevision.revision",
)
runs: Mapped[list["DataflowRun"]] = relationship(
back_populates="pipeline",
cascade="all, delete-orphan",
order_by="DataflowRun.created_at",
)
class DataflowPipelineRevision(Base, TimestampMixin):
__tablename__ = "dataflow_pipeline_revisions"
__table_args__ = (
UniqueConstraint("pipeline_id", "revision", name="uq_dataflow_pipeline_revision"),
Index("ix_dataflow_revisions_tenant_pipeline", "tenant_id", "pipeline_id"),
Index("ix_dataflow_revisions_content_hash", "tenant_id", "content_hash"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
pipeline_id: Mapped[str] = mapped_column(
ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
revision: Mapped[int] = mapped_column(Integer, nullable=False)
schema_version: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
graph: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
sql_text: Mapped[str | None] = mapped_column(Text)
editor_mode: Mapped[str] = mapped_column(String(20), default="graph", nullable=False)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False)
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="revisions")
class DataflowRun(Base, TimestampMixin):
__tablename__ = "dataflow_runs"
__table_args__ = (
Index("ix_dataflow_runs_tenant_status", "tenant_id", "status"),
Index("ix_dataflow_runs_pipeline_created", "pipeline_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
pipeline_id: Mapped[str] = mapped_column(
ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
pipeline_revision_id: Mapped[str] = mapped_column(
ForeignKey("dataflow_pipeline_revisions.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
run_type: Mapped[str] = mapped_column(String(30), default="preview", nullable=False, index=True)
status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
executor_version: Mapped[str] = mapped_column(String(40), nullable=False)
definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
result_schema: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
input_row_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
output_row_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
error: Mapped[str | None] = mapped_column(Text)
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="runs")
__all__ = [
"DataflowPipeline",
"DataflowPipelineRevision",
"DataflowRun",
"new_uuid",
]

312
src/govoplan_dataflow/backend/executor.py Normal file
View File

@@ -0,0 +1,312 @@
from __future__ import annotations
import hashlib
import json
import time
from collections import defaultdict
from dataclasses import dataclass
from decimal import Decimal
from typing import Any
from govoplan_dataflow.backend.graph import graph_input_map, topological_order, validate_graph
from govoplan_dataflow.backend.schemas import (
DataflowDiagnostic,
NodePreviewDiagnostic,
PipelineGraph,
PreviewColumn,
)
EXECUTOR_VERSION = "dataflow-preview-v1"
MAX_EXECUTION_SECONDS = 2.0
MAX_RESULT_BYTES = 1_000_000
class PipelineExecutionError(RuntimeError):
def __init__(self, message: str, *, node_id: str | None = None) -> None:
super().__init__(message)
self.node_id = node_id
@dataclass(frozen=True)
class PipelineExecutionResult:
rows: list[dict[str, Any]]
total_rows: int
truncated: bool
columns: list[PreviewColumn]
diagnostics: list[DataflowDiagnostic]
node_diagnostics: list[NodePreviewDiagnostic]
source_fingerprints: list[dict[str, Any]]
input_row_count: int
def execute_preview(graph: PipelineGraph, *, row_limit: int) -> PipelineExecutionResult:
validation = validate_graph(graph)
errors = [item for item in validation if item.severity == "error"]
if errors:
raise PipelineExecutionError(errors[0].message, node_id=errors[0].node_id)
node_by_id = {node.id: node for node in graph.nodes}
inputs = graph_input_map(graph)
ordered, cyclic = topological_order(graph)
if cyclic:
raise PipelineExecutionError("Pipeline graph contains a cycle")
outputs: dict[str, list[dict[str, Any]]] = {}
node_diagnostics: list[NodePreviewDiagnostic] = []
source_fingerprints: list[dict[str, Any]] = []
started = time.monotonic()
input_row_count = 0
for node_id in ordered:
if time.monotonic() - started > MAX_EXECUTION_SECONDS:
raise PipelineExecutionError("Preview exceeded the two-second execution limit", node_id=node_id)
node = node_by_id[node_id]
node_started = time.monotonic()
input_rows = [] if node.type.startswith("source.") else outputs[inputs[node_id]]
try:
if node.type == "source.inline":
output_rows = [dict(row) for row in node.config.get("rows", [])]
input_row_count += len(output_rows)
source_fingerprints.append(
{
"node_id": node.id,
"source_name": node.config.get("source_name"),
"kind": "inline",
"fingerprint": _rows_fingerprint(output_rows),
"row_count": len(output_rows),
}
)
elif node.type == "source.reference":
raise PipelineExecutionError(
"This connector-backed source is not available to the local preview executor.",
node_id=node.id,
)
elif node.type == "filter":
output_rows = _filter_rows(input_rows, node.config, node_id=node.id)
elif node.type == "select":
output_rows = _select_rows(input_rows, node.config)
elif node.type == "aggregate":
output_rows = _aggregate_rows(input_rows, node.config, node_id=node.id)
elif node.type == "sort":
output_rows = _sort_rows(input_rows, node.config)
elif node.type == "limit":
output_rows = input_rows[: int(node.config["count"])]
elif node.type == "output":
output_rows = [dict(row) for row in input_rows]
else:
raise PipelineExecutionError(f"Unsupported node type: {node.type}", node_id=node.id)
except PipelineExecutionError:
raise
except (ArithmeticError, TypeError, ValueError) as exc:
raise PipelineExecutionError(str(exc), node_id=node.id) from exc
if len(json.dumps(output_rows, default=str).encode("utf-8")) > MAX_RESULT_BYTES:
raise PipelineExecutionError(
"A preview node exceeded the one-megabyte result limit.",
node_id=node.id,
)
outputs[node.id] = output_rows
node_diagnostics.append(
NodePreviewDiagnostic(
node_id=node.id,
status="succeeded",
input_rows=len(input_rows),
output_rows=len(output_rows),
duration_ms=round((time.monotonic() - node_started) * 1000, 3),
columns=infer_columns(output_rows),
)
)
output_node = next(node for node in graph.nodes if node.type == "output")
all_rows = outputs[output_node.id]
rows = all_rows[:row_limit]
return PipelineExecutionResult(
rows=rows,
total_rows=len(all_rows),
truncated=len(rows) < len(all_rows),
columns=infer_columns(all_rows),
diagnostics=[],
node_diagnostics=node_diagnostics,
source_fingerprints=source_fingerprints,
input_row_count=input_row_count,
)
def infer_columns(rows: list[dict[str, Any]]) -> list[PreviewColumn]:
names: list[str] = []
for row in rows:
for name in row:
if name not in names:
names.append(name)
columns: list[PreviewColumn] = []
for name in names:
values = [row.get(name) for row in rows]
concrete = [value for value in values if value is not None]
inferred = _type_name(concrete[0]) if concrete else "unknown"
if any(_type_name(value) != inferred for value in concrete[1:]):
inferred = "mixed"
columns.append(
PreviewColumn(
name=name,
type=inferred,
nullable=len(concrete) != len(values),
)
)
return columns
def _filter_rows(
rows: list[dict[str, Any]],
config: dict[str, Any],
*,
node_id: str,
) -> list[dict[str, Any]]:
column = str(config["column"])
operator = str(config["operator"])
expected = config.get("value")
result: list[dict[str, Any]] = []
for row in rows:
actual = row.get(column)
try:
matches = _compare(actual, operator, expected)
except (TypeError, ValueError) as exc:
raise PipelineExecutionError(
f"Cannot apply {operator!r} to column {column!r}: {exc}",
node_id=node_id,
) from exc
if matches:
result.append(dict(row))
return result
def _compare(actual: Any, operator: str, expected: Any) -> bool:
if operator == "is_null":
return actual is None
if operator == "not_null":
return actual is not None
if operator == "eq":
return actual == expected
if operator == "ne":
return actual != expected
if operator == "contains":
return expected is not None and str(expected).casefold() in str(actual or "").casefold()
if actual is None or expected is None:
return False
if operator == "gt":
return actual > expected
if operator == "gte":
return actual >= expected
if operator == "lt":
return actual < expected
if operator == "lte":
return actual <= expected
raise ValueError(f"unknown operator {operator!r}")
def _select_rows(rows: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]:
fields = config["fields"]
normalized = [
(
field if isinstance(field, str) else str(field["column"]),
field if isinstance(field, str) else str(field.get("alias") or field["column"]),
)
for field in fields
]
return [
{alias: row.get(column) for column, alias in normalized}
for row in rows
]
def _aggregate_rows(
rows: list[dict[str, Any]],
config: dict[str, Any],
*,
node_id: str,
) -> list[dict[str, Any]]:
group_by = [str(item) for item in config.get("group_by", [])]
aggregates = list(config["aggregates"])
grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list)
if rows:
for row in rows:
grouped[tuple(row.get(column) for column in group_by)].append(row)
elif not group_by:
grouped[()] = []
output: list[dict[str, Any]] = []
for key, group_rows in grouped.items():
result = {column: value for column, value in zip(group_by, key, strict=True)}
for aggregate in aggregates:
function = str(aggregate["function"])
column = aggregate.get("column")
alias = str(aggregate["alias"])
values = [row.get(column) for row in group_rows if column is not None and row.get(column) is not None]
try:
if function == "count":
result[alias] = len(group_rows) if column in (None, "", "*") else len(values)
elif function == "sum":
result[alias] = sum(values) if values else 0
elif function == "avg":
result[alias] = sum(values) / len(values) if values else None
elif function == "min":
result[alias] = min(values) if values else None
elif function == "max":
result[alias] = max(values) if values else None
except (ArithmeticError, TypeError, ValueError) as exc:
raise PipelineExecutionError(
f"Cannot calculate {function.upper()} for {column!r}: {exc}",
node_id=node_id,
) from exc
output.append(result)
return output
def _sort_rows(rows: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]:
result = [dict(row) for row in rows]
for field in reversed(config["fields"]):
column = str(field["column"])
reverse = field.get("direction", "asc") == "desc"
result.sort(
key=lambda row: (row.get(column) is None, _sortable_value(row.get(column))),
reverse=reverse,
)
return result
def _sortable_value(value: Any) -> tuple[str, Any]:
if isinstance(value, (int, float, Decimal, str)):
return type(value).__name__, value
return type(value).__name__, str(value)
def _rows_fingerprint(rows: list[dict[str, Any]]) -> str:
encoded = json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _type_name(value: Any) -> str:
if value is None:
return "unknown"
if isinstance(value, bool):
return "boolean"
if isinstance(value, int):
return "integer"
if isinstance(value, (float, Decimal)):
return "number"
if isinstance(value, str):
return "string"
if isinstance(value, list):
return "array"
if isinstance(value, dict):
return "object"
return type(value).__name__.lower()
__all__ = [
"EXECUTOR_VERSION",
"PipelineExecutionError",
"PipelineExecutionResult",
"execute_preview",
"infer_columns",
]

325
src/govoplan_dataflow/backend/graph.py Normal file
View File

@@ -0,0 +1,325 @@
from __future__ import annotations
import hashlib
import json
from collections import deque
from typing import Any
from govoplan_dataflow.backend.schemas import DataflowDiagnostic, GraphNode, PipelineGraph
SUPPORTED_NODE_TYPES = frozenset(
{
"source.inline",
"source.reference",
"filter",
"select",
"aggregate",
"sort",
"limit",
"output",
}
)
FILTER_OPERATORS = frozenset(
{"eq", "ne", "gt", "gte", "lt", "lte", "contains", "is_null", "not_null"}
)
AGGREGATE_FUNCTIONS = frozenset({"count", "sum", "avg", "min", "max"})
def canonical_graph_payload(graph: PipelineGraph) -> dict[str, Any]:
return graph.model_dump(mode="json", exclude_none=True)
def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str:
payload = {
"graph": canonical_graph_payload(graph),
"sql_text": (sql_text or "").strip() or None,
}
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
diagnostics: list[DataflowDiagnostic] = []
nodes = {node.id: node for node in graph.nodes}
if len(nodes) != len(graph.nodes):
diagnostics.append(_error("graph.duplicate_node", "Node identifiers must be unique."))
edge_ids = {edge.id for edge in graph.edges}
if len(edge_ids) != len(graph.edges):
diagnostics.append(_error("graph.duplicate_edge", "Edge identifiers must be unique."))
incoming: dict[str, list[str]] = {node_id: [] for node_id in nodes}
outgoing: dict[str, list[str]] = {node_id: [] for node_id in nodes}
for edge in graph.edges:
if edge.source not in nodes:
diagnostics.append(
_error("edge.unknown_source", f"Edge {edge.id!r} references an unknown source node.")
)
continue
if edge.target not in nodes:
diagnostics.append(
_error("edge.unknown_target", f"Edge {edge.id!r} references an unknown target node.")
)
continue
if edge.source == edge.target:
diagnostics.append(
_error("edge.self_reference", "A node cannot connect to itself.", node_id=edge.source)
)
continue
outgoing[edge.source].append(edge.target)
incoming[edge.target].append(edge.source)
if not graph.nodes:
diagnostics.append(_error("graph.empty", "Add a source and an output before saving the pipeline."))
return diagnostics
source_nodes = [node for node in graph.nodes if node.type.startswith("source.")]
output_nodes = [node for node in graph.nodes if node.type == "output"]
if len(source_nodes) != 1:
diagnostics.append(
_error(
"graph.source_count",
"The first release supports exactly one source node per pipeline.",
)
)
if len(output_nodes) != 1:
diagnostics.append(
_error("graph.output_count", "A pipeline must contain exactly one output node.")
)
for node in graph.nodes:
if node.type not in SUPPORTED_NODE_TYPES:
diagnostics.append(
_error(
"node.unsupported_type",
f"Node type {node.type!r} is not supported by this executor.",
node_id=node.id,
field="type",
)
)
continue
if node.type.startswith("source."):
if incoming.get(node.id):
diagnostics.append(
_error("node.source_has_input", "Source nodes cannot have incoming edges.", node_id=node.id)
)
elif len(incoming.get(node.id, [])) != 1:
diagnostics.append(
_error(
"node.input_count",
"This transform requires exactly one incoming edge.",
node_id=node.id,
)
)
diagnostics.extend(_validate_node_config(node))
ordered, cyclic = topological_order(graph)
if cyclic:
diagnostics.append(_error("graph.cycle", "Pipeline edges must form an acyclic graph."))
elif source_nodes and output_nodes:
reachable = _reachable_from(source_nodes[0].id, outgoing)
if len(reachable) != len(nodes):
diagnostics.append(
_error("graph.disconnected", "Every node must be connected to the pipeline source.")
)
reaches_output = _reachable_from(output_nodes[0].id, incoming)
if len(reaches_output) != len(nodes):
diagnostics.append(
_error("graph.dead_end", "Every node must lead to the pipeline output.")
)
if ordered and ordered[-1] != output_nodes[0].id:
diagnostics.append(
_error("graph.output_not_terminal", "The output node must be the terminal transform.")
)
return diagnostics
def topological_order(graph: PipelineGraph) -> tuple[list[str], bool]:
node_ids = [node.id for node in graph.nodes]
incoming_count = {node_id: 0 for node_id in node_ids}
outgoing: dict[str, list[str]] = {node_id: [] for node_id in node_ids}
for edge in graph.edges:
if edge.source in outgoing and edge.target in incoming_count and edge.source != edge.target:
outgoing[edge.source].append(edge.target)
incoming_count[edge.target] += 1
ready = deque(node_id for node_id in node_ids if incoming_count[node_id] == 0)
ordered: list[str] = []
while ready:
node_id = ready.popleft()
ordered.append(node_id)
for target in outgoing[node_id]:
incoming_count[target] -= 1
if incoming_count[target] == 0:
ready.append(target)
return ordered, len(ordered) != len(node_ids)
def graph_input_map(graph: PipelineGraph) -> dict[str, str]:
return {edge.target: edge.source for edge in graph.edges}
def _reachable_from(start: str, adjacency: dict[str, list[str]]) -> set[str]:
seen: set[str] = set()
pending = [start]
while pending:
node_id = pending.pop()
if node_id in seen:
continue
seen.add(node_id)
pending.extend(adjacency.get(node_id, ()))
return seen
def _validate_node_config(node: GraphNode) -> list[DataflowDiagnostic]:
config = node.config
diagnostics: list[DataflowDiagnostic] = []
if node.type in {"source.inline", "source.reference"}:
source_name = config.get("source_name")
if not isinstance(source_name, str) or not source_name.strip():
diagnostics.append(
_error(
"source.name_required",
"A source needs a logical SQL name.",
node_id=node.id,
field="source_name",
)
)
if node.type == "source.reference":
return diagnostics
rows = config.get("rows")
if not isinstance(rows, list):
diagnostics.append(
_error("source.rows_required", "Inline source rows must be a list.", node_id=node.id, field="rows")
)
elif len(rows) > 250:
diagnostics.append(
_error("source.row_limit", "Inline sources are limited to 250 rows.", node_id=node.id, field="rows")
)
elif any(not isinstance(row, dict) for row in rows):
diagnostics.append(
_error("source.row_shape", "Every inline source row must be an object.", node_id=node.id, field="rows")
)
if len(json.dumps(config, default=str).encode("utf-8")) > 512_000:
diagnostics.append(
_error("source.byte_limit", "Inline source configuration is limited to 512 KiB.", node_id=node.id)
)
elif node.type == "filter":
if not _non_empty_text(config.get("column")):
diagnostics.append(_node_field_error(node, "filter.column", "Choose a column.", "column"))
operator = config.get("operator")
if operator not in FILTER_OPERATORS:
diagnostics.append(
_node_field_error(node, "filter.operator", "Choose a supported comparison operator.", "operator")
)
if operator not in {"is_null", "not_null"} and "value" not in config:
diagnostics.append(_node_field_error(node, "filter.value", "Enter a comparison value.", "value"))
elif node.type == "select":
fields = config.get("fields")
if not isinstance(fields, list) or not fields:
diagnostics.append(
_node_field_error(node, "select.fields", "Select at least one output column.", "fields")
)
else:
for field in fields:
column = field if isinstance(field, str) else field.get("column") if isinstance(field, dict) else None
if not _non_empty_text(column):
diagnostics.append(
_node_field_error(node, "select.field", "Selected fields need a source column.", "fields")
)
break
elif node.type == "aggregate":
group_by = config.get("group_by", [])
aggregates = config.get("aggregates")
if not isinstance(group_by, list) or any(not _non_empty_text(item) for item in group_by):
diagnostics.append(
_node_field_error(node, "aggregate.group_by", "Group-by columns must be named.", "group_by")
)
if not isinstance(aggregates, list) or not aggregates:
diagnostics.append(
_node_field_error(node, "aggregate.required", "Add at least one aggregate.", "aggregates")
)
else:
for aggregate in aggregates:
if not isinstance(aggregate, dict) or aggregate.get("function") not in AGGREGATE_FUNCTIONS:
diagnostics.append(
_node_field_error(
node,
"aggregate.function",
"Choose COUNT, SUM, AVG, MIN, or MAX.",
"aggregates",
)
)
break
if aggregate.get("function") != "count" and not _non_empty_text(aggregate.get("column")):
diagnostics.append(
_node_field_error(
node,
"aggregate.column",
"This aggregate needs a source column.",
"aggregates",
)
)
break
if not _non_empty_text(aggregate.get("alias")):
diagnostics.append(
_node_field_error(node, "aggregate.alias", "Every aggregate needs an alias.", "aggregates")
)
break
elif node.type == "sort":
fields = config.get("fields")
if not isinstance(fields, list) or not fields:
diagnostics.append(_node_field_error(node, "sort.fields", "Add at least one sort field.", "fields"))
elif any(
not isinstance(item, dict)
or not _non_empty_text(item.get("column"))
or item.get("direction", "asc") not in {"asc", "desc"}
for item in fields
):
diagnostics.append(
_node_field_error(node, "sort.field", "Sort fields need a column and direction.", "fields")
)
elif node.type == "limit":
count = config.get("count")
if not isinstance(count, int) or isinstance(count, bool) or not 1 <= count <= 100_000:
diagnostics.append(
_node_field_error(node, "limit.count", "Limit must be between 1 and 100,000.", "count")
)
return diagnostics
def _non_empty_text(value: object) -> bool:
return isinstance(value, str) and bool(value.strip())
def _node_field_error(node: GraphNode, code: str, message: str, field: str) -> DataflowDiagnostic:
return _error(code, message, node_id=node.id, field=field)
def _error(
code: str,
message: str,
*,
node_id: str | None = None,
field: str | None = None,
) -> DataflowDiagnostic:
return DataflowDiagnostic(
severity="error",
code=code,
message=message,
node_id=node_id,
field=field,
)
__all__ = [
"AGGREGATE_FUNCTIONS",
"FILTER_OPERATORS",
"SUPPORTED_NODE_TYPES",
"canonical_graph_payload",
"definition_hash",
"graph_input_map",
"topological_order",
"validate_graph",
]

240
src/govoplan_dataflow/backend/manifest.py Normal file
View File

@@ -0,0 +1,240 @@
from __future__ import annotations
from pathlib import Path
from govoplan_core.core.module_guards import (
drop_table_retirement_provider,
persistent_table_uninstall_guard,
)
from govoplan_core.core.modules import (
DocumentationTopic,
FrontendModule,
MigrationSpec,
ModuleInterfaceProvider,
ModuleManifest,
NavItem,
PermissionDefinition,
RoleTemplate,
)
from govoplan_core.db.base import Base
from govoplan_dataflow.backend.db import models as dataflow_models
MODULE_ID = "dataflow"
MODULE_NAME = "Dataflow"
MODULE_VERSION = "0.1.14"
READ_SCOPE = "dataflow:pipeline:read"
WRITE_SCOPE = "dataflow:pipeline:write"
RUN_SCOPE = "dataflow:pipeline:run"
ADMIN_SCOPE = "dataflow:pipeline:admin"
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
module_id, resource, action = scope.split(":", 2)
return PermissionDefinition(
scope=scope,
label=label,
description=description,
category="Dataflow",
level="tenant",
module_id=module_id,
resource=resource,
action=action,
)
PERMISSIONS = (
_permission(
READ_SCOPE,
"View data pipelines",
"Read pipeline definitions, revisions, validation diagnostics, and run summaries.",
),
_permission(
WRITE_SCOPE,
"Manage data pipelines",
"Create, edit, version, archive, and remove data pipeline definitions.",
),
_permission(
RUN_SCOPE,
"Run data pipelines",
"Validate and execute bounded previews or approved pipeline runs.",
),
_permission(
ADMIN_SCOPE,
"Administer Dataflow",
"Manage every tenant pipeline and future execution, retention, and publication policies.",
),
)
ROLE_TEMPLATES = (
RoleTemplate(
slug="dataflow_designer",
name="Dataflow designer",
description="Design, validate, and preview governed data pipelines.",
permissions=(READ_SCOPE, WRITE_SCOPE, RUN_SCOPE),
),
RoleTemplate(
slug="dataflow_operator",
name="Dataflow operator",
description="Inspect and run approved data pipelines without changing their definitions.",
permissions=(READ_SCOPE, RUN_SCOPE),
),
RoleTemplate(
slug="dataflow_viewer",
name="Dataflow viewer",
description="Inspect pipeline definitions, revisions, diagnostics, and run summaries.",
permissions=(READ_SCOPE,),
),
)
DOCUMENTATION = (
DocumentationTopic(
id="dataflow.module-boundary",
title="Dataflow module boundary",
summary="Versioned tabular transformations with graphical and constrained SQL editing.",
body=(
"Dataflow owns canonical pipeline graphs, immutable revisions, validation, constrained "
"SQL compilation, preview and run diagnostics, and lineage references. Connectors owns "
"source connections and credentials; Reporting owns analytical presentation and exports; "
"Workflow owns orchestration and human handoffs; Risk Compliance owns sanctions review "
"semantics and policy gates. User SQL is compiled into approved transforms and is never "
"passed unchecked to a backing database."
),
layer="available",
documentation_types=("admin", "user"),
audience=("operator", "module_admin", "power_user", "product_owner"),
order=75,
related_modules=(
"connectors",
"files",
"reporting",
"workflow",
"risk_compliance",
"notifications",
"policy",
"audit",
),
metadata={
"first_slice": "Inline source, filter, select, aggregate, sort, limit, output, revisioning, and bounded preview.",
"sql_safety": "Constrained AST compilation only; no pass-through execution.",
},
),
)
def _dataflow_router(_context):
from govoplan_dataflow.backend.router import router
return router
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
return {
"dataflow_pipelines": (
session.query(dataflow_models.DataflowPipeline)
.filter(
dataflow_models.DataflowPipeline.tenant_id == tenant_id,
dataflow_models.DataflowPipeline.deleted_at.is_(None),
)
.count()
),
"dataflow_runs": (
session.query(dataflow_models.DataflowRun)
.filter(dataflow_models.DataflowRun.tenant_id == tenant_id)
.count()
),
}
manifest = ModuleManifest(
id=MODULE_ID,
name=MODULE_NAME,
version=MODULE_VERSION,
dependencies=(),
optional_dependencies=(
"access",
"audit",
"connectors",
"files",
"notifications",
"policy",
"reporting",
"risk_compliance",
"workflow",
),
provides_interfaces=(
ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION),
ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION),
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
ModuleInterfaceProvider(name="dataflow.dataset_output", version=MODULE_VERSION),
),
permissions=PERMISSIONS,
role_templates=ROLE_TEMPLATES,
nav_items=(
NavItem(
path="/dataflow",
label="Dataflow",
icon="waypoints",
required_any=(READ_SCOPE, ADMIN_SCOPE),
order=72,
),
),
frontend=FrontendModule(
module_id=MODULE_ID,
package_name="@govoplan/dataflow-webui",
nav_items=(
NavItem(
path="/dataflow",
label="Dataflow",
icon="waypoints",
required_any=(READ_SCOPE, ADMIN_SCOPE),
order=72,
),
),
),
route_factory=_dataflow_router,
tenant_summary_providers=(_tenant_summary,),
migration_spec=MigrationSpec(
module_id=MODULE_ID,
metadata=Base.metadata,
script_location=str(Path(__file__).with_name("migrations") / "versions"),
retirement_supported=True,
retirement_provider=drop_table_retirement_provider(
dataflow_models.DataflowRun,
dataflow_models.DataflowPipelineRevision,
dataflow_models.DataflowPipeline,
label="Dataflow",
),
retirement_notes=(
"Destructive retirement drops Dataflow definitions, revisions, and run evidence "
"after the installer captures a database snapshot."
),
),
uninstall_guard_providers=(
persistent_table_uninstall_guard(
dataflow_models.DataflowPipeline,
dataflow_models.DataflowPipelineRevision,
dataflow_models.DataflowRun,
label="Dataflow",
),
),
documentation=DOCUMENTATION,
)
def get_manifest() -> ModuleManifest:
return manifest
__all__ = [
"ADMIN_SCOPE",
"MODULE_ID",
"MODULE_NAME",
"MODULE_VERSION",
"READ_SCOPE",
"RUN_SCOPE",
"WRITE_SCOPE",
"get_manifest",
"manifest",
]

1
src/govoplan_dataflow/backend/migrations/__init__.py Normal file
View File

@@ -0,0 +1 @@
from __future__ import annotations

1
src/govoplan_dataflow/backend/migrations/versions/__init__.py Normal file
View File

@@ -0,0 +1 @@
from __future__ import annotations

143
src/govoplan_dataflow/backend/migrations/versions/d4f7a1c8e2b0_v0114_dataflow_baseline.py Normal file
View File

@@ -0,0 +1,143 @@
"""v0.1.14 Dataflow baseline
Revision ID: d4f7a1c8e2b0
Revises: None
Create Date: 2026-07-28 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "d4f7a1c8e2b0"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"dataflow_pipelines",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("name", sa.String(length=300), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("status", sa.String(length=32), nullable=False),
sa.Column("current_revision", sa.Integer(), nullable=False),
sa.Column("created_by", sa.String(length=255), nullable=True),
sa.Column("updated_by", sa.String(length=255), nullable=True),
sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("metadata", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_dataflow_pipelines")),
)
op.create_index(op.f("ix_dataflow_pipelines_created_by"), "dataflow_pipelines", ["created_by"], unique=False)
op.create_index(op.f("ix_dataflow_pipelines_deleted_at"), "dataflow_pipelines", ["deleted_at"], unique=False)
op.create_index(op.f("ix_dataflow_pipelines_status"), "dataflow_pipelines", ["status"], unique=False)
op.create_index(op.f("ix_dataflow_pipelines_tenant_id"), "dataflow_pipelines", ["tenant_id"], unique=False)
op.create_index("ix_dataflow_pipelines_tenant_status", "dataflow_pipelines", ["tenant_id", "status"], unique=False)
op.create_index("ix_dataflow_pipelines_tenant_updated", "dataflow_pipelines", ["tenant_id", "updated_at"], unique=False)
op.create_index(op.f("ix_dataflow_pipelines_updated_by"), "dataflow_pipelines", ["updated_by"], unique=False)
op.create_table(
"dataflow_pipeline_revisions",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("pipeline_id", sa.String(length=36), nullable=False),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("schema_version", sa.Integer(), nullable=False),
sa.Column("graph", sa.JSON(), nullable=False),
sa.Column("sql_text", sa.Text(), nullable=True),
sa.Column("editor_mode", sa.String(length=20), nullable=False),
sa.Column("content_hash", sa.String(length=64), nullable=False),
sa.Column("created_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["pipeline_id"],
["dataflow_pipelines.id"],
name=op.f("fk_dataflow_pipeline_revisions_pipeline_id_dataflow_pipelines"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_dataflow_pipeline_revisions")),
sa.UniqueConstraint("pipeline_id", "revision", name="uq_dataflow_pipeline_revision"),
)
op.create_index(op.f("ix_dataflow_pipeline_revisions_created_by"), "dataflow_pipeline_revisions", ["created_by"], unique=False)
op.create_index(op.f("ix_dataflow_pipeline_revisions_pipeline_id"), "dataflow_pipeline_revisions", ["pipeline_id"], unique=False)
op.create_index(op.f("ix_dataflow_pipeline_revisions_tenant_id"), "dataflow_pipeline_revisions", ["tenant_id"], unique=False)
op.create_index("ix_dataflow_revisions_content_hash", "dataflow_pipeline_revisions", ["tenant_id", "content_hash"], unique=False)
op.create_index("ix_dataflow_revisions_tenant_pipeline", "dataflow_pipeline_revisions", ["tenant_id", "pipeline_id"], unique=False)
op.create_table(
"dataflow_runs",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("pipeline_id", sa.String(length=36), nullable=False),
sa.Column("pipeline_revision_id", sa.String(length=36), nullable=False),
sa.Column("run_type", sa.String(length=30), nullable=False),
sa.Column("status", sa.String(length=30), nullable=False),
sa.Column("executor_version", sa.String(length=40), nullable=False),
sa.Column("definition_hash", sa.String(length=64), nullable=False),
sa.Column("source_fingerprints", sa.JSON(), nullable=False),
sa.Column("result_schema", sa.JSON(), nullable=False),
sa.Column("diagnostics", sa.JSON(), nullable=False),
sa.Column("input_row_count", sa.Integer(), nullable=False),
sa.Column("output_row_count", sa.Integer(), nullable=False),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("created_by", sa.String(length=255), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["pipeline_id"],
["dataflow_pipelines.id"],
name=op.f("fk_dataflow_runs_pipeline_id_dataflow_pipelines"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["pipeline_revision_id"],
["dataflow_pipeline_revisions.id"],
name=op.f("fk_dataflow_runs_pipeline_revision_id_dataflow_pipeline_revisions"),
ondelete="RESTRICT",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_dataflow_runs")),
)
op.create_index(op.f("ix_dataflow_runs_created_by"), "dataflow_runs", ["created_by"], unique=False)
op.create_index(op.f("ix_dataflow_runs_pipeline_id"), "dataflow_runs", ["pipeline_id"], unique=False)
op.create_index(op.f("ix_dataflow_runs_pipeline_revision_id"), "dataflow_runs", ["pipeline_revision_id"], unique=False)
op.create_index(op.f("ix_dataflow_runs_run_type"), "dataflow_runs", ["run_type"], unique=False)
op.create_index(op.f("ix_dataflow_runs_status"), "dataflow_runs", ["status"], unique=False)
op.create_index(op.f("ix_dataflow_runs_tenant_id"), "dataflow_runs", ["tenant_id"], unique=False)
op.create_index("ix_dataflow_runs_pipeline_created", "dataflow_runs", ["pipeline_id", "created_at"], unique=False)
op.create_index("ix_dataflow_runs_tenant_status", "dataflow_runs", ["tenant_id", "status"], unique=False)
def downgrade() -> None:
op.drop_index("ix_dataflow_runs_tenant_status", table_name="dataflow_runs")
op.drop_index("ix_dataflow_runs_pipeline_created", table_name="dataflow_runs")
op.drop_index(op.f("ix_dataflow_runs_tenant_id"), table_name="dataflow_runs")
op.drop_index(op.f("ix_dataflow_runs_status"), table_name="dataflow_runs")
op.drop_index(op.f("ix_dataflow_runs_run_type"), table_name="dataflow_runs")
op.drop_index(op.f("ix_dataflow_runs_pipeline_revision_id"), table_name="dataflow_runs")
op.drop_index(op.f("ix_dataflow_runs_pipeline_id"), table_name="dataflow_runs")
op.drop_index(op.f("ix_dataflow_runs_created_by"), table_name="dataflow_runs")
op.drop_table("dataflow_runs")
op.drop_index("ix_dataflow_revisions_tenant_pipeline", table_name="dataflow_pipeline_revisions")
op.drop_index("ix_dataflow_revisions_content_hash", table_name="dataflow_pipeline_revisions")
op.drop_index(op.f("ix_dataflow_pipeline_revisions_tenant_id"), table_name="dataflow_pipeline_revisions")
op.drop_index(op.f("ix_dataflow_pipeline_revisions_pipeline_id"), table_name="dataflow_pipeline_revisions")
op.drop_index(op.f("ix_dataflow_pipeline_revisions_created_by"), table_name="dataflow_pipeline_revisions")
op.drop_table("dataflow_pipeline_revisions")
op.drop_index(op.f("ix_dataflow_pipelines_updated_by"), table_name="dataflow_pipelines")
op.drop_index("ix_dataflow_pipelines_tenant_updated", table_name="dataflow_pipelines")
op.drop_index("ix_dataflow_pipelines_tenant_status", table_name="dataflow_pipelines")
op.drop_index(op.f("ix_dataflow_pipelines_tenant_id"), table_name="dataflow_pipelines")
op.drop_index(op.f("ix_dataflow_pipelines_status"), table_name="dataflow_pipelines")
op.drop_index(op.f("ix_dataflow_pipelines_deleted_at"), table_name="dataflow_pipelines")
op.drop_index(op.f("ix_dataflow_pipelines_created_by"), table_name="dataflow_pipelines")
op.drop_table("dataflow_pipelines")

264
src/govoplan_dataflow/backend/router.py Normal file
View File

@@ -0,0 +1,264 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
from govoplan_core.db.session import get_session
from govoplan_dataflow.backend.manifest import ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, WRITE_SCOPE
from govoplan_dataflow.backend.schemas import (
PipelineCreateRequest,
PipelineDeleteResponse,
PipelineDraftRequest,
PipelineListResponse,
PipelinePreviewRequest,
PipelinePreviewResponse,
PipelineResponse,
PipelineSqlResponse,
PipelineUpdateRequest,
PipelineValidationResponse,
)
from govoplan_dataflow.backend.service import (
DataflowConflictError,
DataflowError,
DataflowNotFoundError,
DataflowValidationError,
compile_sql_draft,
create_pipeline,
delete_pipeline,
get_pipeline,
list_pipelines,
pipeline_response,
preview_pipeline,
render_graph_sql,
update_pipeline,
validate_draft,
)
router = APIRouter(prefix="/dataflow", tags=["dataflow"])
def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None:
if any(has_scope(principal, scope) for scope in scopes):
return
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"Missing one of the required scopes: {', '.join(scopes)}",
)
def _actor_id(principal: ApiPrincipal) -> str | None:
return (
principal.account_id
or principal.membership_id
or getattr(principal.user, "id", None)
or principal.identity_id
)
def _http_error(exc: DataflowError) -> HTTPException:
if isinstance(exc, DataflowNotFoundError):
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
if isinstance(exc, DataflowConflictError):
return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
if isinstance(exc, DataflowValidationError):
return HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail={
"message": str(exc),
"diagnostics": [item.model_dump(mode="json") for item in exc.diagnostics],
},
)
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
@router.get("/pipelines", response_model=PipelineListResponse)
def api_list_pipelines(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineListResponse:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
pipelines = list_pipelines(session, tenant_id=principal.tenant_id)
return PipelineListResponse(
pipelines=[pipeline_response(session, pipeline) for pipeline in pipelines]
)
@router.post("/pipelines", response_model=PipelineResponse, status_code=status.HTTP_201_CREATED)
def api_create_pipeline(
payload: PipelineCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
pipeline = create_pipeline(
session,
tenant_id=principal.tenant_id,
actor_id=_actor_id(principal),
payload=payload,
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.pipeline.created",
object_type="dataflow_pipeline",
object_id=pipeline.id,
details={"revision": pipeline.current_revision, "status": pipeline.status},
)
response = pipeline_response(session, pipeline)
session.commit()
return response
@router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse)
def api_get_pipeline(
pipeline_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineResponse:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
try:
pipeline = get_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
return pipeline_response(session, pipeline)
except DataflowError as exc:
raise _http_error(exc) from exc
@router.put("/pipelines/{pipeline_id}", response_model=PipelineResponse)
def api_update_pipeline(
pipeline_id: str,
payload: PipelineUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
pipeline = update_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
actor_id=_actor_id(principal),
payload=payload,
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.pipeline.updated",
object_type="dataflow_pipeline",
object_id=pipeline.id,
details={"revision": pipeline.current_revision, "status": pipeline.status},
)
response = pipeline_response(session, pipeline)
session.commit()
return response
@router.delete("/pipelines/{pipeline_id}", response_model=PipelineDeleteResponse)
def api_delete_pipeline(
pipeline_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineDeleteResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
pipeline = delete_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
actor_id=_actor_id(principal),
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.pipeline.deleted",
object_type="dataflow_pipeline",
object_id=pipeline.id,
details={"revision": pipeline.current_revision},
)
session.commit()
return PipelineDeleteResponse(deleted=True, pipeline_id=pipeline.id)
@router.post("/validate", response_model=PipelineValidationResponse)
def api_validate_pipeline(
payload: PipelineDraftRequest,
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineValidationResponse:
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
return validate_draft(payload)
@router.post("/sql/compile", response_model=PipelineSqlResponse)
def api_compile_pipeline_sql(
payload: PipelineDraftRequest,
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineSqlResponse:
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
return compile_sql_draft(payload)
@router.post("/sql/render", response_model=PipelineSqlResponse)
def api_render_pipeline_sql(
payload: PipelineDraftRequest,
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineSqlResponse:
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
return render_graph_sql(payload)
@router.post("/preview", response_model=PipelinePreviewResponse)
def api_preview_pipeline(
payload: PipelinePreviewRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelinePreviewResponse:
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
try:
response = preview_pipeline(
session,
tenant_id=principal.tenant_id,
actor_id=_actor_id(principal),
payload=payload,
)
except DataflowError as exc:
raise _http_error(exc) from exc
if response.pipeline_id:
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.pipeline.previewed",
object_type="dataflow_pipeline",
object_id=response.pipeline_id,
details={
"revision": response.revision,
"run_id": response.run_id,
"status": response.status,
"output_row_count": response.total_rows,
},
)
session.commit()
return response
__all__ = ["router"]

161
src/govoplan_dataflow/backend/schemas.py Normal file
View File

@@ -0,0 +1,161 @@
from __future__ import annotations
import math
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
PipelineStatus = Literal["draft", "active", "archived"]
EditorMode = Literal["graph", "sql"]
DiagnosticSeverity = Literal["info", "warning", "error"]
class GraphPosition(BaseModel):
x: float
y: float
@field_validator("x", "y")
@classmethod
def finite_coordinate(cls, value: float) -> float:
if not math.isfinite(value):
raise ValueError("Graph coordinates must be finite")
return value
class GraphNode(BaseModel):
id: str = Field(min_length=1, max_length=100, pattern=r"^[A-Za-z0-9_.:-]+$")
type: str = Field(min_length=1, max_length=80)
label: str = Field(min_length=1, max_length=200)
position: GraphPosition
config: dict[str, Any] = Field(default_factory=dict)
class GraphEdge(BaseModel):
id: str = Field(min_length=1, max_length=120, pattern=r"^[A-Za-z0-9_.:-]+$")
source: str = Field(min_length=1, max_length=100)
target: str = Field(min_length=1, max_length=100)
source_port: str = Field(default="output", min_length=1, max_length=80)
target_port: str = Field(default="input", min_length=1, max_length=80)
class PipelineGraph(BaseModel):
schema_version: Literal[1] = 1
nodes: list[GraphNode] = Field(default_factory=list, max_length=100)
edges: list[GraphEdge] = Field(default_factory=list, max_length=200)
class DataflowDiagnostic(BaseModel):
severity: DiagnosticSeverity
code: str
message: str
node_id: str | None = None
field: str | None = None
class PipelineRevisionResponse(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
revision: int
schema_version: int
graph: PipelineGraph
sql_text: str | None
editor_mode: EditorMode
content_hash: str
created_by: str | None
created_at: datetime
class PipelineResponse(BaseModel):
id: str
tenant_id: str
name: str
description: str | None
status: PipelineStatus
current_revision: int
created_by: str | None
updated_by: str | None
created_at: datetime
updated_at: datetime
revision: PipelineRevisionResponse
class PipelineListResponse(BaseModel):
pipelines: list[PipelineResponse]
class PipelineCreateRequest(BaseModel):
name: str = Field(min_length=1, max_length=300)
description: str | None = Field(default=None, max_length=4000)
status: PipelineStatus = "draft"
graph: PipelineGraph
sql_text: str | None = Field(default=None, max_length=100_000)
editor_mode: EditorMode = "graph"
class PipelineUpdateRequest(PipelineCreateRequest):
expected_revision: int = Field(ge=1)
class PipelineDraftRequest(BaseModel):
graph: PipelineGraph | None = None
sql_text: str | None = Field(default=None, max_length=100_000)
source_nodes: list[GraphNode] = Field(default_factory=list, max_length=20)
class PipelineValidationResponse(BaseModel):
valid: bool
graph: PipelineGraph | None
sql_text: str | None
diagnostics: list[DataflowDiagnostic]
class PipelineSqlResponse(PipelineValidationResponse):
pass
class PipelinePreviewRequest(BaseModel):
pipeline_id: str | None = None
revision: int | None = Field(default=None, ge=1)
graph: PipelineGraph | None = None
sql_text: str | None = Field(default=None, max_length=100_000)
source_nodes: list[GraphNode] = Field(default_factory=list, max_length=20)
row_limit: int = Field(default=100, ge=1, le=500)
class PreviewColumn(BaseModel):
name: str
type: str
nullable: bool = True
class NodePreviewDiagnostic(BaseModel):
node_id: str
status: Literal["succeeded", "failed", "skipped"]
input_rows: int = 0
output_rows: int = 0
duration_ms: float = 0
columns: list[PreviewColumn] = Field(default_factory=list)
messages: list[str] = Field(default_factory=list)
class PipelinePreviewResponse(BaseModel):
run_id: str | None
pipeline_id: str | None
revision: int | None
status: Literal["succeeded", "failed"]
columns: list[PreviewColumn]
rows: list[dict[str, Any]]
total_rows: int
truncated: bool
diagnostics: list[DataflowDiagnostic]
node_diagnostics: list[NodePreviewDiagnostic]
definition_hash: str
executor_version: str
class PipelineDeleteResponse(BaseModel):
deleted: bool
pipeline_id: str

511
src/govoplan_dataflow/backend/service.py Normal file
View File

@@ -0,0 +1,511 @@
from __future__ import annotations
from dataclasses import dataclass
from sqlalchemy import select
from sqlalchemy.orm import Session
from govoplan_core.db.base import utcnow
from govoplan_dataflow.backend.db.models import (
DataflowPipeline,
DataflowPipelineRevision,
DataflowRun,
)
from govoplan_dataflow.backend.executor import (
EXECUTOR_VERSION,
PipelineExecutionError,
execute_preview,
)
from govoplan_dataflow.backend.graph import canonical_graph_payload, definition_hash, validate_graph
from govoplan_dataflow.backend.schemas import (
DataflowDiagnostic,
GraphNode,
PipelineCreateRequest,
PipelineDraftRequest,
PipelineGraph,
PipelinePreviewRequest,
PipelinePreviewResponse,
PipelineResponse,
PipelineRevisionResponse,
PipelineSqlResponse,
PipelineUpdateRequest,
PipelineValidationResponse,
)
from govoplan_dataflow.backend.sql_compiler import (
SqlCompilationError,
compile_sql,
render_sql,
)
class DataflowError(RuntimeError):
pass
class DataflowNotFoundError(DataflowError):
pass
class DataflowConflictError(DataflowError):
pass
class DataflowValidationError(DataflowError):
def __init__(self, diagnostics: list[DataflowDiagnostic]) -> None:
super().__init__(diagnostics[0].message if diagnostics else "Pipeline validation failed")
self.diagnostics = diagnostics
@dataclass(frozen=True)
class NormalizedDefinition:
graph: PipelineGraph
sql_text: str | None
diagnostics: list[DataflowDiagnostic]
def list_pipelines(session: Session, *, tenant_id: str) -> list[DataflowPipeline]:
return list(
session.scalars(
select(DataflowPipeline)
.where(
DataflowPipeline.tenant_id == tenant_id,
DataflowPipeline.deleted_at.is_(None),
)
.order_by(DataflowPipeline.updated_at.desc(), DataflowPipeline.name)
)
)
def get_pipeline(
session: Session,
*,
tenant_id: str,
pipeline_id: str,
) -> DataflowPipeline:
pipeline = session.scalar(
select(DataflowPipeline).where(
DataflowPipeline.id == pipeline_id,
DataflowPipeline.tenant_id == tenant_id,
DataflowPipeline.deleted_at.is_(None),
)
)
if pipeline is None:
raise DataflowNotFoundError("Dataflow pipeline not found")
return pipeline
def get_pipeline_revision(
session: Session,
*,
pipeline: DataflowPipeline,
revision: int | None = None,
) -> DataflowPipelineRevision:
revision_number = revision or pipeline.current_revision
item = session.scalar(
select(DataflowPipelineRevision).where(
DataflowPipelineRevision.pipeline_id == pipeline.id,
DataflowPipelineRevision.tenant_id == pipeline.tenant_id,
DataflowPipelineRevision.revision == revision_number,
)
)
if item is None:
raise DataflowNotFoundError("Dataflow pipeline revision not found")
return item
def create_pipeline(
session: Session,
*,
tenant_id: str,
actor_id: str | None,
payload: PipelineCreateRequest,
) -> DataflowPipeline:
definition = normalize_definition(
graph=payload.graph,
sql_text=payload.sql_text,
editor_mode=payload.editor_mode,
)
content_hash = definition_hash(definition.graph, definition.sql_text)
pipeline = DataflowPipeline(
tenant_id=tenant_id,
name=payload.name.strip(),
description=_clean_optional(payload.description),
status=payload.status,
current_revision=1,
created_by=actor_id,
updated_by=actor_id,
metadata_={},
)
revision = DataflowPipelineRevision(
tenant_id=tenant_id,
revision=1,
schema_version=definition.graph.schema_version,
graph=canonical_graph_payload(definition.graph),
sql_text=definition.sql_text,
editor_mode=payload.editor_mode,
content_hash=content_hash,
created_by=actor_id,
)
pipeline.revisions.append(revision)
session.add(pipeline)
session.flush()
return pipeline
def update_pipeline(
session: Session,
*,
tenant_id: str,
pipeline_id: str,
actor_id: str | None,
payload: PipelineUpdateRequest,
) -> DataflowPipeline:
pipeline = get_pipeline(session, tenant_id=tenant_id, pipeline_id=pipeline_id)
if payload.expected_revision != pipeline.current_revision:
raise DataflowConflictError(
f"Pipeline changed on the server; expected revision {payload.expected_revision}, "
f"current revision is {pipeline.current_revision}"
)
definition = normalize_definition(
graph=payload.graph,
sql_text=payload.sql_text,
editor_mode=payload.editor_mode,
)
content_hash = definition_hash(definition.graph, definition.sql_text)
current = get_pipeline_revision(session, pipeline=pipeline)
pipeline.name = payload.name.strip()
pipeline.description = _clean_optional(payload.description)
pipeline.status = payload.status
pipeline.updated_by = actor_id
if current.content_hash != content_hash or current.editor_mode != payload.editor_mode:
pipeline.current_revision += 1
pipeline.revisions.append(
DataflowPipelineRevision(
tenant_id=tenant_id,
revision=pipeline.current_revision,
schema_version=definition.graph.schema_version,
graph=canonical_graph_payload(definition.graph),
sql_text=definition.sql_text,
editor_mode=payload.editor_mode,
content_hash=content_hash,
created_by=actor_id,
)
)
session.flush()
return pipeline
def delete_pipeline(
session: Session,
*,
tenant_id: str,
pipeline_id: str,
actor_id: str | None,
) -> DataflowPipeline:
pipeline = get_pipeline(session, tenant_id=tenant_id, pipeline_id=pipeline_id)
pipeline.deleted_at = utcnow()
pipeline.updated_by = actor_id
session.flush()
return pipeline
def pipeline_response(session: Session, pipeline: DataflowPipeline) -> PipelineResponse:
revision = get_pipeline_revision(session, pipeline=pipeline)
return PipelineResponse(
id=pipeline.id,
tenant_id=pipeline.tenant_id,
name=pipeline.name,
description=pipeline.description,
status=pipeline.status,
current_revision=pipeline.current_revision,
created_by=pipeline.created_by,
updated_by=pipeline.updated_by,
created_at=pipeline.created_at,
updated_at=pipeline.updated_at,
revision=PipelineRevisionResponse.model_validate(revision),
)
def validate_draft(payload: PipelineDraftRequest) -> PipelineValidationResponse:
if payload.sql_text and payload.sql_text.strip():
try:
graph, sql_text, diagnostics = compile_sql(
payload.sql_text,
source_nodes=_source_nodes(payload.graph, payload.source_nodes),
)
except SqlCompilationError as exc:
return PipelineValidationResponse(
valid=False,
graph=payload.graph,
sql_text=payload.sql_text,
diagnostics=exc.diagnostics,
)
return PipelineValidationResponse(
valid=True,
graph=graph,
sql_text=sql_text,
diagnostics=diagnostics,
)
if payload.graph is None:
diagnostic = DataflowDiagnostic(
severity="error",
code="definition.required",
message="Provide a graph or SQL query.",
)
return PipelineValidationResponse(
valid=False,
graph=None,
sql_text=None,
diagnostics=[diagnostic],
)
diagnostics = validate_graph(payload.graph)
sql_text: str | None = None
if not any(item.severity == "error" for item in diagnostics):
try:
sql_text, render_diagnostics = render_sql(payload.graph)
diagnostics.extend(render_diagnostics)
except SqlCompilationError as exc:
diagnostics.extend(
DataflowDiagnostic(
severity="warning",
code=item.code,
message=item.message,
node_id=item.node_id,
field=item.field,
)
for item in exc.diagnostics
)
return PipelineValidationResponse(
valid=not any(item.severity == "error" for item in diagnostics),
graph=payload.graph,
sql_text=sql_text,
diagnostics=diagnostics,
)
def compile_sql_draft(payload: PipelineDraftRequest) -> PipelineSqlResponse:
if not payload.sql_text:
diagnostic = DataflowDiagnostic(
severity="error",
code="sql.empty",
message="Enter a SELECT query.",
field="sql_text",
)
return PipelineSqlResponse(valid=False, graph=payload.graph, sql_text="", diagnostics=[diagnostic])
try:
graph, sql_text, diagnostics = compile_sql(
payload.sql_text,
source_nodes=_source_nodes(payload.graph, payload.source_nodes),
)
except SqlCompilationError as exc:
return PipelineSqlResponse(
valid=False,
graph=payload.graph,
sql_text=payload.sql_text,
diagnostics=exc.diagnostics,
)
return PipelineSqlResponse(valid=True, graph=graph, sql_text=sql_text, diagnostics=diagnostics)
def render_graph_sql(payload: PipelineDraftRequest) -> PipelineSqlResponse:
if payload.graph is None:
diagnostic = DataflowDiagnostic(
severity="error",
code="graph.required",
message="Provide a graph to render.",
)
return PipelineSqlResponse(valid=False, graph=None, sql_text=None, diagnostics=[diagnostic])
try:
sql_text, diagnostics = render_sql(payload.graph)
except SqlCompilationError as exc:
return PipelineSqlResponse(
valid=False,
graph=payload.graph,
sql_text=None,
diagnostics=exc.diagnostics,
)
return PipelineSqlResponse(valid=True, graph=payload.graph, sql_text=sql_text, diagnostics=diagnostics)
def preview_pipeline(
session: Session,
*,
tenant_id: str,
actor_id: str | None,
payload: PipelinePreviewRequest,
) -> PipelinePreviewResponse:
pipeline: DataflowPipeline | None = None
revision: DataflowPipelineRevision | None = None
if payload.pipeline_id:
pipeline = get_pipeline(session, tenant_id=tenant_id, pipeline_id=payload.pipeline_id)
revision = get_pipeline_revision(session, pipeline=pipeline, revision=payload.revision)
graph = PipelineGraph.model_validate(revision.graph)
sql_text = revision.sql_text
else:
draft = PipelineDraftRequest(
graph=payload.graph,
sql_text=payload.sql_text,
source_nodes=payload.source_nodes,
)
validated = validate_draft(draft)
if not validated.valid or validated.graph is None:
return PipelinePreviewResponse(
run_id=None,
pipeline_id=None,
revision=None,
status="failed",
columns=[],
rows=[],
total_rows=0,
truncated=False,
diagnostics=validated.diagnostics,
node_diagnostics=[],
definition_hash="",
executor_version=EXECUTOR_VERSION,
)
graph = validated.graph
sql_text = validated.sql_text
graph_hash = definition_hash(graph, sql_text)
started_at = utcnow()
run: DataflowRun | None = None
try:
result = execute_preview(graph, row_limit=payload.row_limit)
status = "succeeded"
error = None
diagnostics = result.diagnostics
columns = result.columns
rows = result.rows
total_rows = result.total_rows
truncated = result.truncated
node_diagnostics = result.node_diagnostics
source_fingerprints = result.source_fingerprints
input_row_count = result.input_row_count
except PipelineExecutionError as exc:
status = "failed"
error = str(exc)
diagnostics = [
DataflowDiagnostic(
severity="error",
code="preview.execution",
message=str(exc),
node_id=exc.node_id,
)
]
columns = []
rows = []
total_rows = 0
truncated = False
node_diagnostics = []
source_fingerprints = []
input_row_count = 0
if pipeline is not None and revision is not None:
run = DataflowRun(
tenant_id=tenant_id,
pipeline_id=pipeline.id,
pipeline_revision_id=revision.id,
run_type="preview",
status=status,
executor_version=EXECUTOR_VERSION,
definition_hash=graph_hash,
source_fingerprints=source_fingerprints,
result_schema=[item.model_dump(mode="json") for item in columns],
diagnostics=[item.model_dump(mode="json") for item in diagnostics],
input_row_count=input_row_count,
output_row_count=total_rows,
started_at=started_at,
finished_at=utcnow(),
error=error,
created_by=actor_id,
)
session.add(run)
session.flush()
return PipelinePreviewResponse(
run_id=run.id if run else None,
pipeline_id=pipeline.id if pipeline else None,
revision=revision.revision if revision else None,
status=status,
columns=columns,
rows=rows,
total_rows=total_rows,
truncated=truncated,
diagnostics=diagnostics,
node_diagnostics=node_diagnostics,
definition_hash=graph_hash,
executor_version=EXECUTOR_VERSION,
)
def normalize_definition(
*,
graph: PipelineGraph,
sql_text: str | None,
editor_mode: str,
) -> NormalizedDefinition:
if editor_mode == "sql":
try:
compiled_graph, normalized_sql, diagnostics = compile_sql(
sql_text or "",
source_nodes=_source_nodes(graph, ()),
)
except SqlCompilationError as exc:
raise DataflowValidationError(exc.diagnostics) from exc
return NormalizedDefinition(
graph=compiled_graph,
sql_text=normalized_sql,
diagnostics=diagnostics,
)
diagnostics = validate_graph(graph)
errors = [item for item in diagnostics if item.severity == "error"]
if errors:
raise DataflowValidationError(diagnostics)
try:
rendered_sql, render_diagnostics = render_sql(graph)
diagnostics.extend(render_diagnostics)
except SqlCompilationError:
rendered_sql = None
return NormalizedDefinition(graph=graph, sql_text=rendered_sql, diagnostics=diagnostics)
def _source_nodes(
graph: PipelineGraph | None,
explicit_nodes: list[GraphNode] | tuple[()],
) -> list[GraphNode]:
nodes = list(explicit_nodes)
if graph is not None:
known = {node.id for node in nodes}
nodes.extend(
node
for node in graph.nodes
if node.type.startswith("source.") and node.id not in known
)
return nodes
def _clean_optional(value: str | None) -> str | None:
if value is None:
return None
cleaned = value.strip()
return cleaned or None
__all__ = [
"DataflowConflictError",
"DataflowError",
"DataflowNotFoundError",
"DataflowValidationError",
"compile_sql_draft",
"create_pipeline",
"delete_pipeline",
"get_pipeline",
"get_pipeline_revision",
"list_pipelines",
"normalize_definition",
"pipeline_response",
"preview_pipeline",
"render_graph_sql",
"update_pipeline",
"validate_draft",
]

483
src/govoplan_dataflow/backend/sql_compiler.py Normal file
View File

@@ -0,0 +1,483 @@
from __future__ import annotations
from typing import Any, Iterable
import sqlglot
from sqlglot import exp
from sqlglot.errors import ParseError
from govoplan_dataflow.backend.graph import topological_order, validate_graph
from govoplan_dataflow.backend.schemas import (
DataflowDiagnostic,
GraphEdge,
GraphNode,
GraphPosition,
PipelineGraph,
)
class SqlCompilationError(ValueError):
def __init__(self, diagnostics: list[DataflowDiagnostic]) -> None:
super().__init__(diagnostics[0].message if diagnostics else "SQL compilation failed")
self.diagnostics = diagnostics
def compile_sql(
sql_text: str,
*,
source_nodes: Iterable[GraphNode] = (),
) -> tuple[PipelineGraph, str, list[DataflowDiagnostic]]:
source_text = sql_text.strip()
if not source_text:
raise SqlCompilationError([_sql_error("sql.empty", "Enter a SELECT query.")])
try:
statements = sqlglot.parse(source_text, read="duckdb")
except ParseError as exc:
detail = exc.errors[0] if exc.errors else {}
line = detail.get("line")
column = detail.get("col")
location = f" at line {line}, column {column}" if line and column else ""
raise SqlCompilationError(
[_sql_error("sql.parse", f"SQL could not be parsed{location}: {detail.get('description') or exc}")]
) from exc
if len(statements) != 1 or not isinstance(statements[0], exp.Select):
raise SqlCompilationError(
[_sql_error("sql.select_only", "Dataflow SQL accepts exactly one SELECT statement.")]
)
query = statements[0]
_reject_unsupported_query_shape(query)
tables = list(query.find_all(exp.Table))
if len(tables) != 1:
raise SqlCompilationError(
[_sql_error("sql.source_count", "The first release supports exactly one logical source.")]
)
table = tables[0]
if table.catalog or table.db:
raise SqlCompilationError(
[_sql_error("sql.qualified_source", "Use the logical source name without a catalog or schema.")]
)
source_name = table.name
preserved_source = next(
(
node.model_copy(deep=True)
for node in source_nodes
if node.type.startswith("source.")
and str(node.config.get("source_name", "")).casefold() == source_name.casefold()
),
None,
)
source_node = preserved_source or GraphNode(
id="source",
type="source.reference",
label=source_name,
position=GraphPosition(x=80, y=180),
config={"source_name": source_name},
)
nodes = [source_node]
conditions = _flatten_and(query.args.get("where").this) if query.args.get("where") else []
for index, condition in enumerate(conditions, start=1):
nodes.append(
GraphNode(
id=f"filter-{index}",
type="filter",
label=f"Filter {index}",
position=_position(len(nodes)),
config=_condition_config(condition),
)
)
group = query.args.get("group")
group_by = [_column_name(item, context="GROUP BY") for item in group.expressions] if group else []
aggregate_specs: list[dict[str, Any]] = []
projection_fields: list[dict[str, str]] = []
saw_star = False
saw_aggregate = False
for item in query.expressions:
inner = item.this if isinstance(item, exp.Alias) else item
alias = item.alias if isinstance(item, exp.Alias) else ""
aggregate = _aggregate_config(inner, alias=alias)
if aggregate is not None:
saw_aggregate = True
aggregate_specs.append(aggregate)
continue
if isinstance(inner, exp.Star):
saw_star = True
continue
if not isinstance(inner, exp.Column):
raise SqlCompilationError(
[_sql_error("sql.select_expression", "SELECT supports columns and COUNT/SUM/AVG/MIN/MAX only.")]
)
column = _column_name(inner, context="SELECT")
projection_fields.append({"column": column, "alias": alias or column})
if saw_star and len(query.expressions) != 1:
raise SqlCompilationError(
[_sql_error("sql.star_mix", "SELECT * cannot be mixed with other expressions in this dialect.")]
)
if saw_aggregate or group_by:
if saw_star:
raise SqlCompilationError([_sql_error("sql.aggregate_star", "SELECT * cannot be grouped.")])
plain_columns = [field["column"] for field in projection_fields]
if any(column not in group_by for column in plain_columns):
raise SqlCompilationError(
[_sql_error("sql.grouping", "Every non-aggregate SELECT column must appear in GROUP BY.")]
)
if any(field["alias"] != field["column"] for field in projection_fields):
raise SqlCompilationError(
[_sql_error("sql.group_alias", "Aliases for GROUP BY columns are not supported yet.")]
)
if not aggregate_specs:
raise SqlCompilationError(
[_sql_error("sql.aggregate_required", "GROUP BY requires at least one aggregate in this dialect.")]
)
nodes.append(
GraphNode(
id="aggregate",
type="aggregate",
label="Aggregate",
position=_position(len(nodes)),
config={"group_by": group_by, "aggregates": aggregate_specs},
)
)
elif not saw_star:
nodes.append(
GraphNode(
id="select",
type="select",
label="Select columns",
position=_position(len(nodes)),
config={"fields": projection_fields},
)
)
order = query.args.get("order")
if order:
fields: list[dict[str, str]] = []
for item in order.expressions:
if not isinstance(item, exp.Ordered):
raise SqlCompilationError([_sql_error("sql.order", "Unsupported ORDER BY expression.")])
fields.append(
{
"column": _column_name(item.this, context="ORDER BY"),
"direction": "desc" if item.args.get("desc") else "asc",
}
)
nodes.append(
GraphNode(
id="sort",
type="sort",
label="Sort",
position=_position(len(nodes)),
config={"fields": fields},
)
)
limit = query.args.get("limit")
if limit:
expression = limit.args.get("expression")
if not isinstance(expression, exp.Literal) or expression.is_string:
raise SqlCompilationError([_sql_error("sql.limit", "LIMIT must be a positive integer.")])
try:
count = int(expression.this)
except (TypeError, ValueError) as exc:
raise SqlCompilationError([_sql_error("sql.limit", "LIMIT must be a positive integer.")]) from exc
if not 1 <= count <= 100_000:
raise SqlCompilationError(
[_sql_error("sql.limit_range", "LIMIT must be between 1 and 100,000.")]
)
nodes.append(
GraphNode(
id="limit",
type="limit",
label="Limit",
position=_position(len(nodes)),
config={"count": count},
)
)
nodes.append(
GraphNode(
id="output",
type="output",
label="Preview output",
position=_position(len(nodes)),
config={},
)
)
edges = [
GraphEdge(
id=f"edge-{source.id}-{target.id}",
source=source.id,
target=target.id,
)
for source, target in zip(nodes, nodes[1:])
]
graph = PipelineGraph(nodes=nodes, edges=edges)
diagnostics = validate_graph(graph)
if any(item.severity == "error" for item in diagnostics):
raise SqlCompilationError(diagnostics)
return graph, query.sql(dialect="duckdb", pretty=True), diagnostics
def render_sql(graph: PipelineGraph) -> tuple[str, list[DataflowDiagnostic]]:
diagnostics = validate_graph(graph)
if any(item.severity == "error" for item in diagnostics):
raise SqlCompilationError(diagnostics)
ordered, cyclic = topological_order(graph)
if cyclic:
raise SqlCompilationError([_sql_error("graph.cycle", "A cyclic graph cannot be rendered as SQL.")])
node_by_id = {node.id: node for node in graph.nodes}
source = node_by_id[ordered[0]]
source_name = str(source.config.get("source_name", "")).strip()
if not source_name:
raise SqlCompilationError([_sql_error("source.name_required", "The source needs a logical SQL name.")])
where_conditions: list[exp.Expression] = []
select_expressions: list[exp.Expression] = [exp.Star()]
group_by: list[exp.Expression] = []
order_by: list[exp.Expression] = []
limit: int | None = None
selected = False
for node_id in ordered[1:]:
node = node_by_id[node_id]
if node.type == "filter":
if selected:
raise SqlCompilationError(
[_node_sql_error(node.id, "sql.filter_order", "Filters after projection or aggregation are not representable yet.")]
)
where_conditions.append(_condition_expression(node.config))
elif node.type == "select":
if selected:
raise SqlCompilationError(
[_node_sql_error(node.id, "sql.multiple_select", "Only one select or aggregate transform is supported.")]
)
select_expressions = []
for field in node.config["fields"]:
column = field if isinstance(field, str) else str(field["column"])
alias = column if isinstance(field, str) else str(field.get("alias") or column)
expression: exp.Expression = exp.column(column)
if alias != column:
expression = expression.as_(alias)
select_expressions.append(expression)
selected = True
elif node.type == "aggregate":
if selected:
raise SqlCompilationError(
[_node_sql_error(node.id, "sql.multiple_select", "Only one select or aggregate transform is supported.")]
)
select_expressions = [exp.column(column) for column in node.config.get("group_by", [])]
group_by = [exp.column(column) for column in node.config.get("group_by", [])]
for aggregate in node.config["aggregates"]:
function = str(aggregate["function"])
column = aggregate.get("column")
argument: exp.Expression = exp.Star() if function == "count" and column in (None, "", "*") else exp.column(str(column))
aggregate_expression = _aggregate_expression(function, argument)
select_expressions.append(aggregate_expression.as_(str(aggregate["alias"])))
selected = True
elif node.type == "sort":
order_by = [
exp.Ordered(
this=exp.column(str(field["column"])),
desc=field.get("direction", "asc") == "desc",
nulls_first=False,
)
for field in node.config["fields"]
]
elif node.type == "limit":
limit = int(node.config["count"])
elif node.type != "output":
raise SqlCompilationError(
[_node_sql_error(node.id, "sql.node_not_representable", f"{node.type!r} cannot be rendered as SQL.")]
)
query = exp.select(*select_expressions).from_(exp.to_table(source_name))
if where_conditions:
query = query.where(_combine_and(where_conditions))
if group_by:
query = query.group_by(*group_by)
if order_by:
query = query.order_by(*order_by)
if limit is not None:
query = query.limit(limit)
return query.sql(dialect="duckdb", pretty=True), diagnostics
def _reject_unsupported_query_shape(query: exp.Select) -> None:
unsupported_args = {
"with_": "WITH queries",
"distinct": "DISTINCT",
"having": "HAVING",
"qualify": "QUALIFY",
"offset": "OFFSET",
"windows": "window definitions",
"locks": "locking clauses",
}
for key, label in unsupported_args.items():
if query.args.get(key):
raise SqlCompilationError(
[_sql_error("sql.unsupported_clause", f"{label} are not supported by the first Dataflow dialect.")]
)
if any(True for _ in query.find_all(exp.Join)):
raise SqlCompilationError([_sql_error("sql.join", "JOIN support belongs to the comparison/reconciliation slice.")])
if any(True for _ in query.find_all(exp.Subquery)):
raise SqlCompilationError([_sql_error("sql.subquery", "Subqueries are not supported by the first Dataflow dialect.")])
def _flatten_and(expression: exp.Expression) -> list[exp.Expression]:
if isinstance(expression, exp.And):
return [*_flatten_and(expression.this), *_flatten_and(expression.expression)]
if isinstance(expression, exp.Or):
raise SqlCompilationError([_sql_error("sql.or", "OR conditions are not supported yet; use separate pipelines.")])
return [expression]
def _condition_config(expression: exp.Expression) -> dict[str, Any]:
if isinstance(expression, exp.Not) and isinstance(expression.this, exp.Is):
inner = expression.this
if isinstance(inner.this, exp.Column) and isinstance(inner.expression, exp.Null):
return {"column": _column_name(inner.this, context="WHERE"), "operator": "not_null"}
if isinstance(expression, exp.Is):
if isinstance(expression.this, exp.Column) and isinstance(expression.expression, exp.Null):
return {"column": _column_name(expression.this, context="WHERE"), "operator": "is_null"}
mapping: tuple[tuple[type[exp.Expression], str], ...] = (
(exp.EQ, "eq"),
(exp.NEQ, "ne"),
(exp.GT, "gt"),
(exp.GTE, "gte"),
(exp.LT, "lt"),
(exp.LTE, "lte"),
)
for expression_type, operator in mapping:
if isinstance(expression, expression_type):
if not isinstance(expression.this, exp.Column):
break
return {
"column": _column_name(expression.this, context="WHERE"),
"operator": operator,
"value": _literal_value(expression.expression),
}
if isinstance(expression, exp.Like) and isinstance(expression.this, exp.Column):
value = _literal_value(expression.expression)
if not isinstance(value, str) or not (value.startswith("%") and value.endswith("%")):
raise SqlCompilationError(
[_sql_error("sql.like", "LIKE is supported only as a contains pattern: LIKE '%value%'.")]
)
return {
"column": _column_name(expression.this, context="WHERE"),
"operator": "contains",
"value": value[1:-1],
}
raise SqlCompilationError(
[_sql_error("sql.condition", "WHERE supports simple column comparisons joined with AND.")]
)
def _condition_expression(config: dict[str, Any]) -> exp.Expression:
column = exp.column(str(config["column"]))
operator = str(config["operator"])
if operator == "is_null":
return exp.Is(this=column, expression=exp.Null())
if operator == "not_null":
return exp.Not(this=exp.Is(this=column, expression=exp.Null()))
value = exp.convert(config.get("value"))
mapping: dict[str, type[exp.Expression]] = {
"eq": exp.EQ,
"ne": exp.NEQ,
"gt": exp.GT,
"gte": exp.GTE,
"lt": exp.LT,
"lte": exp.LTE,
}
if operator in mapping:
return mapping[operator](this=column, expression=value)
if operator == "contains":
return exp.Like(this=column, expression=exp.Literal.string(f"%{config.get('value', '')}%"))
raise SqlCompilationError([_sql_error("filter.operator", f"Unsupported filter operator {operator!r}.")])
def _literal_value(expression: exp.Expression) -> Any:
if isinstance(expression, exp.Null):
return None
if isinstance(expression, exp.Boolean):
return bool(expression.this)
if not isinstance(expression, exp.Literal):
raise SqlCompilationError([_sql_error("sql.literal", "Comparisons require a literal value.")])
if expression.is_string:
return str(expression.this)
text = str(expression.this)
try:
return int(text)
except ValueError:
try:
return float(text)
except ValueError as exc:
raise SqlCompilationError([_sql_error("sql.literal", f"Unsupported literal {text!r}.")]) from exc
def _aggregate_config(expression: exp.Expression, *, alias: str) -> dict[str, Any] | None:
mapping: tuple[tuple[type[exp.Expression], str], ...] = (
(exp.Count, "count"),
(exp.Sum, "sum"),
(exp.Avg, "avg"),
(exp.Min, "min"),
(exp.Max, "max"),
)
for expression_type, function in mapping:
if not isinstance(expression, expression_type):
continue
argument = expression.this
if isinstance(argument, exp.Star):
column = "*"
elif isinstance(argument, exp.Column):
column = _column_name(argument, context=function.upper())
else:
raise SqlCompilationError(
[_sql_error("sql.aggregate_argument", f"{function.upper()} requires a column or * argument.")]
)
output_alias = alias or (f"{function}_all" if column == "*" else f"{function}_{column}")
return {"function": function, "column": column, "alias": output_alias}
return None
def _aggregate_expression(function: str, argument: exp.Expression) -> exp.Expression:
mapping: dict[str, type[exp.Expression]] = {
"count": exp.Count,
"sum": exp.Sum,
"avg": exp.Avg,
"min": exp.Min,
"max": exp.Max,
}
return mapping[function](this=argument)
def _column_name(expression: exp.Expression, *, context: str) -> str:
if not isinstance(expression, exp.Column) or expression.table:
raise SqlCompilationError(
[_sql_error("sql.column", f"{context} accepts unqualified column names only.")]
)
return expression.name
def _position(index: int) -> GraphPosition:
return GraphPosition(x=80 + index * 220, y=180)
def _combine_and(expressions: list[exp.Expression]) -> exp.Expression:
combined = expressions[0]
for expression in expressions[1:]:
combined = exp.And(this=combined, expression=expression)
return combined
def _sql_error(code: str, message: str) -> DataflowDiagnostic:
return DataflowDiagnostic(severity="error", code=code, message=message, field="sql_text")
def _node_sql_error(node_id: str, code: str, message: str) -> DataflowDiagnostic:
return DataflowDiagnostic(severity="error", code=code, message=message, node_id=node_id)
__all__ = ["SqlCompilationError", "compile_sql", "render_sql"]