Add durable reconciliation row evidence

This commit is contained in:
2026-08-04 11:38:29 +02:00
parent 42f87ddc88
commit 3991c179a4
7 changed files with 257 additions and 11 deletions
+63 -5
View File
@@ -878,12 +878,32 @@ def _reconcile_rows(
right_keys = [str(item) for item in config["right_keys"]]
right_prefix = str(config.get("right_prefix", "observed_"))
comparisons = _comparison_fields(config.get("compare_columns"))
left_index = _unique_row_index(left_rows, left_keys, node_id=node_id)
right_index = _unique_row_index(right_rows, right_keys, node_id=node_id)
left_index = _unique_row_index(
left_rows,
left_keys,
node_id=node_id,
input_label="expected",
)
right_index = _unique_row_index(
right_rows,
right_keys,
node_id=node_id,
input_label="observed",
)
output: list[dict[str, Any]] = []
for key in dict.fromkeys((*left_index, *right_index)):
left = left_index.get(key)
right = right_index.get(key)
key_values = [
(left or {}).get(left_name)
if left is not None
else (right or {}).get(right_name)
for left_name, right_name in zip(
left_keys,
right_keys,
strict=True,
)
]
item = dict(left or {})
if right is not None:
item.update({f"{right_prefix}{name}": value for name, value in right.items()})
@@ -895,18 +915,55 @@ def _reconcile_rows(
differences = []
else:
fields = comparisons or tuple((name, name) for name in left if name not in left_keys)
differences = [
left_name
changes = [
{
"expected_field": left_name,
"observed_field": right_name,
"expected": left.get(left_name),
"observed": right.get(right_name),
}
for left_name, right_name in fields
if left.get(left_name) != right.get(right_name)
]
differences = [str(change["expected_field"]) for change in changes]
status = "changed" if differences else "match"
if left is None or right is None:
changes = []
item["_reconciliation_status"] = status
item["_reconciliation_differences"] = differences
item["_reconciliation_changes"] = changes
item["_reconciliation_key"] = key_values
item["_reconciliation_key_hash"] = _reconciliation_hash(
{
"left_keys": left_keys,
"right_keys": right_keys,
"values": key_values,
}
)
item["_reconciliation_input_hash"] = _reconciliation_hash(
{
"key": key_values,
"expected": left,
"observed": right,
"comparisons": comparisons,
}
)
item["_reconciliation_before"] = dict(left) if left is not None else None
item["_reconciliation_after"] = dict(right) if right is not None else None
output.append(item)
return output
def _reconciliation_hash(value: object) -> str:
encoded = json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
default=str,
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _comparison_fields(value: object) -> tuple[tuple[str, str], ...]:
if not isinstance(value, list):
return ()
@@ -927,13 +984,14 @@ def _unique_row_index(
columns: list[str],
*,
node_id: str,
input_label: str,
) -> dict[tuple[Any, ...], dict[str, Any]]:
index: dict[tuple[Any, ...], dict[str, Any]] = {}
for row in rows:
key = tuple(_hashable(row.get(column)) for column in columns)
if key in index:
raise PipelineExecutionError(
f"Reconciliation keys are not unique: {key!r}.",
f"Reconciliation keys are not unique in the {input_label} input.",
node_id=node_id,
)
index[key] = row
@@ -200,6 +200,7 @@ DOCUMENTATION = (
"Every graph node declares typed inputs, configuration, output schema, and validation rules. "
"Source nodes pin inline content or governed Datasource references; combine, filter, transform, "
"quality, reconciliation, reusable-subflow, and output nodes remain explicit in the canonical graph. "
"Reconciliation rows expose stable key hashes, explicit before/after values, and input hashes so a later governed human decision can be replayed only while its inputs still match. "
"Expressions use the typed Dataflow expression language and never execute arbitrary host or database "
"code. Selecting a node may request a bounded intermediate preview; preview rows are transient, "
"privacy-filtered for the actor, and are not retained as run output. SQL editing compiles into the same "
@@ -510,7 +510,7 @@ _NODE_TYPES = (
type="reconcile.compare",
category="quality",
label="Reconcile tables",
description="Compare keyed rows and expose missing, matching, and changed records.",
description="Compare keyed rows with stable identity, before/after evidence, and decision invalidation fingerprints.",
icon="scan-search",
input_ports=(
NodePortDefinition(id="left", label="Expected"),
@@ -612,6 +612,12 @@ def _reconcile(context: SchemaPropagationContext) -> SchemaPropagationResult:
(
"_reconciliation_status",
"_reconciliation_differences",
"_reconciliation_changes",
"_reconciliation_key",
"_reconciliation_key_hash",
"_reconciliation_input_hash",
"_reconciliation_before",
"_reconciliation_after",
)
),
open=left_state.open or right_state.open,
@@ -623,6 +629,12 @@ def _reconcile(context: SchemaPropagationContext) -> SchemaPropagationResult:
},
"_reconciliation_status": "string",
"_reconciliation_differences": "array",
"_reconciliation_changes": "array",
"_reconciliation_key": "array",
"_reconciliation_key_hash": "string",
"_reconciliation_input_hash": "string",
"_reconciliation_before": "object",
"_reconciliation_after": "object",
},
)
return SchemaPropagationResult(state, tuple(diagnostics))