Preserve dataflow layout across SQL edits
This commit is contained in:
@@ -55,6 +55,74 @@ def definition_hash(graph: PipelineGraph, sql_text: str | None = None) -> str:
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def preserve_compatible_graph_layout(
|
||||
reference: PipelineGraph,
|
||||
compiled: PipelineGraph,
|
||||
) -> PipelineGraph:
|
||||
"""Keep graph identity and layout when SQL preserves the same topology."""
|
||||
reference_keys = _structural_node_keys(reference)
|
||||
compiled_keys = _structural_node_keys(compiled)
|
||||
if reference_keys is None or compiled_keys is None:
|
||||
return compiled
|
||||
if (
|
||||
len(set(reference_keys.values())) != len(reference_keys)
|
||||
or len(set(compiled_keys.values())) != len(compiled_keys)
|
||||
or set(reference_keys.values()) != set(compiled_keys.values())
|
||||
):
|
||||
return compiled
|
||||
|
||||
compiled_by_key = {
|
||||
key: next(node for node in compiled.nodes if node.id == node_id)
|
||||
for node_id, key in compiled_keys.items()
|
||||
}
|
||||
reference_edges = {
|
||||
_structural_edge_key(edge, reference_keys): edge
|
||||
for edge in reference.edges
|
||||
}
|
||||
compiled_edges = {
|
||||
_structural_edge_key(edge, compiled_keys): edge
|
||||
for edge in compiled.edges
|
||||
}
|
||||
if (
|
||||
len(reference_edges) != len(reference.edges)
|
||||
or len(compiled_edges) != len(compiled.edges)
|
||||
or set(reference_edges) != set(compiled_edges)
|
||||
):
|
||||
return compiled
|
||||
|
||||
nodes = []
|
||||
for reference_node in reference.nodes:
|
||||
node = compiled_by_key[reference_keys[reference_node.id]]
|
||||
nodes.append(
|
||||
node.model_copy(
|
||||
update={
|
||||
"id": reference_node.id,
|
||||
"label": reference_node.label,
|
||||
"position": reference_node.position.model_copy(deep=True),
|
||||
},
|
||||
deep=True,
|
||||
)
|
||||
)
|
||||
edges = []
|
||||
for reference_edge in reference.edges:
|
||||
edge_key = _structural_edge_key(reference_edge, reference_keys)
|
||||
edge = compiled_edges[edge_key]
|
||||
edges.append(
|
||||
edge.model_copy(
|
||||
update={
|
||||
"id": reference_edge.id,
|
||||
"source": reference_edge.source,
|
||||
"target": reference_edge.target,
|
||||
},
|
||||
deep=True,
|
||||
)
|
||||
)
|
||||
return compiled.model_copy(
|
||||
update={"nodes": nodes, "edges": edges},
|
||||
deep=True,
|
||||
)
|
||||
|
||||
|
||||
def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
diagnostics = [
|
||||
DataflowDiagnostic(
|
||||
@@ -151,6 +219,70 @@ def validate_graph(graph: PipelineGraph) -> list[DataflowDiagnostic]:
|
||||
return _dedupe_diagnostics(diagnostics)
|
||||
|
||||
|
||||
def _structural_node_keys(
|
||||
graph: PipelineGraph,
|
||||
) -> dict[str, tuple[object, ...]] | None:
|
||||
node_by_id = {node.id: node for node in graph.nodes}
|
||||
if (
|
||||
len(node_by_id) != len(graph.nodes)
|
||||
or any(
|
||||
edge.source not in node_by_id or edge.target not in node_by_id
|
||||
for edge in graph.edges
|
||||
)
|
||||
):
|
||||
return None
|
||||
ordered, cyclic = topological_order(graph)
|
||||
if cyclic:
|
||||
return None
|
||||
incoming: dict[str, list[GraphEdge]] = {
|
||||
node.id: []
|
||||
for node in graph.nodes
|
||||
}
|
||||
for edge in graph.edges:
|
||||
if edge.target in incoming:
|
||||
incoming[edge.target].append(edge)
|
||||
keys: dict[str, tuple[object, ...]] = {}
|
||||
for node_id in ordered:
|
||||
node = node_by_id[node_id]
|
||||
input_keys: list[tuple[object, ...]] = []
|
||||
for edge in incoming[node_id]:
|
||||
source_key = keys.get(edge.source)
|
||||
if source_key is None:
|
||||
return None
|
||||
input_keys.append(
|
||||
(
|
||||
edge.target_port,
|
||||
edge.source_port,
|
||||
source_key,
|
||||
)
|
||||
)
|
||||
if node.type != "combine.union":
|
||||
input_keys.sort(key=repr)
|
||||
source_name = (
|
||||
str(node.config.get("source_name", "")).strip().casefold()
|
||||
if node.type.startswith("source.")
|
||||
else None
|
||||
)
|
||||
keys[node_id] = (
|
||||
node.type,
|
||||
source_name,
|
||||
tuple(input_keys),
|
||||
)
|
||||
return keys
|
||||
|
||||
|
||||
def _structural_edge_key(
|
||||
edge: GraphEdge,
|
||||
node_keys: dict[str, tuple[object, ...]],
|
||||
) -> tuple[object, ...]:
|
||||
return (
|
||||
node_keys[edge.source],
|
||||
node_keys[edge.target],
|
||||
edge.source_port,
|
||||
edge.target_port,
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_diagnostics(
|
||||
diagnostics: list[DataflowDiagnostic],
|
||||
) -> list[DataflowDiagnostic]:
|
||||
|
||||
Reference in New Issue
Block a user