311 lines
9.3 KiB
TypeScript
311 lines
9.3 KiB
TypeScript
import {
|
|
parseCsv,
|
|
safeJsonParse,
|
|
type JsonValue,
|
|
} from "@add-ideas/toolbox-helpers";
|
|
import { DOMParser, type Element as XmlElement } from "@xmldom/xmldom";
|
|
|
|
export type DataFormat = "json" | "csv" | "ndjson" | "xml";
|
|
export type CsvInferenceMode = "none" | "safe" | "aggressive";
|
|
export interface ParseDataOptions {
|
|
csvInference?: CsvInferenceMode;
|
|
inferNulls?: boolean;
|
|
trimBeforeInference?: boolean;
|
|
rejectUnsafeJsonNumbers?: boolean;
|
|
}
|
|
export interface CsvCellEvidence {
|
|
row: number;
|
|
column: number;
|
|
field: string;
|
|
raw: string;
|
|
value: JsonValue;
|
|
inferred: "string" | "number" | "boolean" | "null";
|
|
}
|
|
export interface JsonNumberEvidence {
|
|
offset: number;
|
|
raw: string;
|
|
parsed: number;
|
|
risk: "unsafe-integer" | "precision-risk";
|
|
}
|
|
export interface DataEvidence {
|
|
csv?: {
|
|
headers: string[];
|
|
rawRows: string[][];
|
|
cells: CsvCellEvidence[];
|
|
inference: {
|
|
csvInference: CsvInferenceMode;
|
|
inferNulls: boolean;
|
|
trimBeforeInference: boolean;
|
|
};
|
|
};
|
|
jsonNumbers: JsonNumberEvidence[];
|
|
}
|
|
export interface DataDocument {
|
|
format: DataFormat;
|
|
root: JsonValue;
|
|
rows: JsonValue[];
|
|
fields: string[];
|
|
evidence: DataEvidence;
|
|
}
|
|
const MAX_CHARS = 2 * 1024 * 1024,
|
|
MAX_ROWS = 10_000,
|
|
MAX_FIELDS = 200;
|
|
|
|
export function parseData(
|
|
source: string,
|
|
format: DataFormat,
|
|
options: ParseDataOptions = {},
|
|
): DataDocument {
|
|
if (source.length > MAX_CHARS)
|
|
throw new Error("Input exceeds the 2 MiB limit.");
|
|
const evidence: DataEvidence = {
|
|
jsonNumbers:
|
|
format === "json" || format === "ndjson" ? scanJsonNumbers(source) : [],
|
|
};
|
|
if (options.rejectUnsafeJsonNumbers && evidence.jsonNumbers.length)
|
|
throw new RangeError(
|
|
`JSON contains ${evidence.jsonNumbers.length} number token${evidence.jsonNumbers.length === 1 ? "" : "s"} that cannot be accepted without precision risk.`,
|
|
);
|
|
let root: JsonValue;
|
|
if (format === "json")
|
|
root = safeJsonParse(source, {
|
|
maxTextChars: MAX_CHARS,
|
|
maxDepth: 32,
|
|
maxNodes: 200_000,
|
|
});
|
|
else if (format === "ndjson") root = parseNdjson(source);
|
|
else if (format === "csv") {
|
|
const csv = parseCsvData(source, options);
|
|
root = csv.root;
|
|
evidence.csv = csv.evidence;
|
|
} else root = parseXml(source);
|
|
const rows = normalizeRows(root);
|
|
if (rows.length > MAX_ROWS)
|
|
throw new Error("Dataset exceeds 10,000 query rows.");
|
|
const fields = [
|
|
...new Set(
|
|
rows.flatMap((row) => (isObject(row) ? Object.keys(row) : ["value"])),
|
|
),
|
|
];
|
|
if (fields.length > MAX_FIELDS)
|
|
throw new Error("Dataset exposes more than 200 top-level fields.");
|
|
return { format, root, rows, fields: fields.sort(), evidence };
|
|
}
|
|
|
|
function parseCsvData(
|
|
source: string,
|
|
options: ParseDataOptions,
|
|
): { root: JsonValue; evidence: NonNullable<DataEvidence["csv"]> } {
|
|
const table = parseCsv(source, {
|
|
maxRows: MAX_ROWS + 1,
|
|
maxColumns: MAX_FIELDS,
|
|
maxFieldChars: 100_000,
|
|
});
|
|
const inference = {
|
|
csvInference: options.csvInference ?? "safe",
|
|
inferNulls: options.inferNulls ?? false,
|
|
trimBeforeInference: options.trimBeforeInference ?? true,
|
|
};
|
|
if (!table.length)
|
|
return {
|
|
root: [],
|
|
evidence: { headers: [], rawRows: [], cells: [], inference },
|
|
};
|
|
const headers = table[0]!.map(
|
|
(value, index) => value.trim() || `column_${index + 1}`,
|
|
);
|
|
if (new Set(headers).size !== headers.length)
|
|
throw new Error("CSV headers must be unique.");
|
|
const rawRows = table.slice(1).filter((row) => row.some(Boolean));
|
|
const cells: CsvCellEvidence[] = [];
|
|
const root = rawRows.map((row, rowIndex) =>
|
|
Object.fromEntries(
|
|
headers.map((field, column) => {
|
|
const raw = row[column] ?? "";
|
|
const interpreted = inferCsvScalar(raw, inference);
|
|
cells.push({
|
|
row: rowIndex + 1,
|
|
column: column + 1,
|
|
field,
|
|
raw,
|
|
value: interpreted.value,
|
|
inferred: interpreted.kind,
|
|
});
|
|
return [field, interpreted.value];
|
|
}),
|
|
),
|
|
) as JsonValue;
|
|
return {
|
|
root,
|
|
evidence: {
|
|
headers: [...headers],
|
|
rawRows: rawRows.map((row) => [...row]),
|
|
cells,
|
|
inference,
|
|
},
|
|
};
|
|
}
|
|
|
|
function inferCsvScalar(
|
|
raw: string,
|
|
options: {
|
|
csvInference: CsvInferenceMode;
|
|
inferNulls: boolean;
|
|
trimBeforeInference: boolean;
|
|
},
|
|
): { value: JsonValue; kind: CsvCellEvidence["inferred"] } {
|
|
if (options.csvInference === "none") return { value: raw, kind: "string" };
|
|
const candidate = options.trimBeforeInference ? raw.trim() : raw;
|
|
if (/^(?:true|false)$/iu.test(candidate))
|
|
return { value: candidate.toLowerCase() === "true", kind: "boolean" };
|
|
if (options.inferNulls && /^null$/iu.test(candidate))
|
|
return { value: null, kind: "null" };
|
|
if (/^-?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/iu.test(candidate)) {
|
|
const number = Number(candidate);
|
|
const integerToken = /^-?\d+$/u.test(candidate);
|
|
if (
|
|
Number.isFinite(number) &&
|
|
(options.csvInference === "aggressive" ||
|
|
!integerToken ||
|
|
Number.isSafeInteger(number))
|
|
)
|
|
return { value: number, kind: "number" };
|
|
}
|
|
return { value: raw, kind: "string" };
|
|
}
|
|
|
|
/** Identifies JSON number lexemes whose browser Number conversion loses evidence. */
|
|
export function scanJsonNumbers(source: string): JsonNumberEvidence[] {
|
|
const results: JsonNumberEvidence[] = [];
|
|
let index = 0;
|
|
let inString = false;
|
|
while (index < source.length) {
|
|
const char = source[index]!;
|
|
if (inString) {
|
|
if (char === "\\") index += 2;
|
|
else {
|
|
if (char === '"') inString = false;
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (char === '"') {
|
|
inString = true;
|
|
index += 1;
|
|
continue;
|
|
}
|
|
const match = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/u.exec(
|
|
source.slice(index),
|
|
);
|
|
if (!match) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
const raw = match[0];
|
|
const parsed = Number(raw);
|
|
const integer = !/[.eE]/u.test(raw);
|
|
const mantissa = raw.split(/[eE]/u, 1)[0] ?? raw;
|
|
const significantDigits = mantissa
|
|
.replace(/^-|\./gu, "")
|
|
.replace(/^0+/u, "")
|
|
.replace(/0+$/u, "").length;
|
|
const risk = integer
|
|
? Number.isSafeInteger(parsed)
|
|
? undefined
|
|
: "unsafe-integer"
|
|
: !Number.isFinite(parsed) || significantDigits > 15
|
|
? "precision-risk"
|
|
: undefined;
|
|
if (risk) results.push({ offset: index, raw, parsed, risk });
|
|
index += raw.length;
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function parseNdjson(source: string): JsonValue {
|
|
const lines = source.split(/\r?\n/u).filter((line) => line.trim());
|
|
if (lines.length > MAX_ROWS)
|
|
throw new Error("NDJSON exceeds 10,000 records.");
|
|
return lines.map((line, index) => {
|
|
try {
|
|
return safeJsonParse(line, {
|
|
maxTextChars: 200_000,
|
|
maxDepth: 32,
|
|
maxNodes: 20_000,
|
|
});
|
|
} catch (error) {
|
|
throw new Error(
|
|
`NDJSON line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
function parseXml(source: string): JsonValue {
|
|
if (/<!\s*(?:DOCTYPE|ENTITY)\b/iu.test(source))
|
|
throw new Error("XML DTD and entity declarations are not accepted.");
|
|
const errors: string[] = [];
|
|
const document = new DOMParser({
|
|
onError(level, message) {
|
|
if (level !== "warning") errors.push(message);
|
|
},
|
|
}).parseFromString(source, "application/xml");
|
|
if (
|
|
errors.length ||
|
|
document.getElementsByTagName("parsererror").length ||
|
|
!document.documentElement
|
|
)
|
|
throw new Error(`Malformed XML${errors[0] ? `: ${errors[0]}` : "."}`);
|
|
let nodes = 0;
|
|
const convert = (element: XmlElement, depth: number): JsonValue => {
|
|
if (++nodes > 100_000 || depth > 32)
|
|
throw new Error("XML structure exceeds node/depth limits.");
|
|
const output: Record<string, JsonValue> = {};
|
|
for (let index = 0; index < element.attributes.length; index += 1) {
|
|
const attribute = element.attributes.item(index);
|
|
if (attribute) output[`@${attribute.name}`] = attribute.value;
|
|
}
|
|
const allChildren = Array.from(
|
|
{ length: element.childNodes.length },
|
|
(_, index) => element.childNodes.item(index),
|
|
);
|
|
const children = allChildren.filter((node): node is XmlElement =>
|
|
Boolean(node && node.nodeType === 1),
|
|
);
|
|
for (const child of children) {
|
|
const value = convert(child, depth + 1);
|
|
const existing = output[child.nodeName];
|
|
output[child.nodeName] =
|
|
existing === undefined
|
|
? value
|
|
: Array.isArray(existing)
|
|
? [...existing, value]
|
|
: [existing, value];
|
|
}
|
|
const text = allChildren
|
|
.filter((node) => node?.nodeType === 3 || node?.nodeType === 4)
|
|
.map((node) => node?.nodeValue ?? "")
|
|
.join("")
|
|
.trim();
|
|
if (!Object.keys(output).length) return text;
|
|
if (text) output["#text"] = text;
|
|
return output;
|
|
};
|
|
return {
|
|
[document.documentElement.nodeName]: convert(document.documentElement, 0),
|
|
};
|
|
}
|
|
|
|
export function normalizeRows(value: JsonValue): JsonValue[] {
|
|
if (Array.isArray(value)) return value;
|
|
if (isObject(value)) {
|
|
const arrays = Object.values(value).filter(Array.isArray);
|
|
if (arrays.length === 1) return arrays[0] as JsonValue[];
|
|
}
|
|
return [value];
|
|
}
|
|
export function isObject(value: JsonValue): value is Record<string, JsonValue> {
|
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
}
|