feat: harden datasource materialization payloads

This commit is contained in:
2026-07-29 19:38:34 +02:00
parent 5f0847b8ca
commit f3288dbd28
9 changed files with 1460 additions and 179 deletions
+29 -5
View File
@@ -111,11 +111,33 @@ def fingerprint_rows(
def encoded_size(rows: Sequence[Mapping[str, object]]) -> int:
return len(
json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str).encode(
"utf-8"
)
)
return len(_encoded_rows(rows))
def payload_checksum(rows: Sequence[Mapping[str, object]]) -> str:
return hashlib.sha256(_encoded_rows(rows)).hexdigest()
def payload_row_checksum(row: Mapping[str, object]) -> str:
return hashlib.sha256(_encoded_row(row)).hexdigest()
def _encoded_rows(rows: Sequence[Mapping[str, object]]) -> bytes:
return json.dumps(
rows,
sort_keys=True,
separators=(",", ":"),
default=str,
).encode("utf-8")
def _encoded_row(row: Mapping[str, object]) -> bytes:
return json.dumps(
row,
sort_keys=True,
separators=(",", ":"),
default=str,
).encode("utf-8")
def field_payload(field: DatasourceField) -> dict[str, object]:
@@ -194,5 +216,7 @@ __all__ = [
"fingerprint_rows",
"infer_schema",
"normalize_rows",
"payload_checksum",
"payload_row_checksum",
"parse_csv_rows",
]