feat: harden datasource materialization payloads
This commit is contained in:
@@ -111,11 +111,33 @@ def fingerprint_rows(
|
||||
|
||||
|
||||
def encoded_size(rows: Sequence[Mapping[str, object]]) -> int:
|
||||
return len(
|
||||
json.dumps(rows, sort_keys=True, separators=(",", ":"), default=str).encode(
|
||||
"utf-8"
|
||||
)
|
||||
)
|
||||
return len(_encoded_rows(rows))
|
||||
|
||||
|
||||
def payload_checksum(rows: Sequence[Mapping[str, object]]) -> str:
|
||||
return hashlib.sha256(_encoded_rows(rows)).hexdigest()
|
||||
|
||||
|
||||
def payload_row_checksum(row: Mapping[str, object]) -> str:
|
||||
return hashlib.sha256(_encoded_row(row)).hexdigest()
|
||||
|
||||
|
||||
def _encoded_rows(rows: Sequence[Mapping[str, object]]) -> bytes:
|
||||
return json.dumps(
|
||||
rows,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _encoded_row(row: Mapping[str, object]) -> bytes:
|
||||
return json.dumps(
|
||||
row,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def field_payload(field: DatasourceField) -> dict[str, object]:
|
||||
@@ -194,5 +216,7 @@ __all__ = [
|
||||
"fingerprint_rows",
|
||||
"infer_schema",
|
||||
"normalize_rows",
|
||||
"payload_checksum",
|
||||
"payload_row_checksum",
|
||||
"parse_csv_rows",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user