Release Query Tools 0.2.0

2026-09-02 10:28:20 +02:00
parent 2730ce08b2
commit f44d0598da
34 changed files with 5153 additions and 300 deletions
@@ -6,19 +6,66 @@ import {
 import { DOMParser, type Element as XmlElement } from "@xmldom/xmldom";

 export type DataFormat = "json" | "csv" | "ndjson" | "xml";
+export type CsvInferenceMode = "none" | "safe" | "aggressive";
+export interface ParseDataOptions {
+  csvInference?: CsvInferenceMode;
+  inferNulls?: boolean;
+  trimBeforeInference?: boolean;
+  rejectUnsafeJsonNumbers?: boolean;
+}
+export interface CsvCellEvidence {
+  row: number;
+  column: number;
+  field: string;
+  raw: string;
+  value: JsonValue;
+  inferred: "string" | "number" | "boolean" | "null";
+}
+export interface JsonNumberEvidence {
+  offset: number;
+  raw: string;
+  parsed: number;
+  risk: "unsafe-integer" | "precision-risk";
+}
+export interface DataEvidence {
+  csv?: {
+    headers: string[];
+    rawRows: string[][];
+    cells: CsvCellEvidence[];
+    inference: {
+      csvInference: CsvInferenceMode;
+      inferNulls: boolean;
+      trimBeforeInference: boolean;
+    };
+  };
+  jsonNumbers: JsonNumberEvidence[];
+}
 export interface DataDocument {
  format: DataFormat;
  root: JsonValue;
  rows: JsonValue[];
  fields: string[];
+  evidence: DataEvidence;
 }
 const MAX_CHARS = 2 * 1024 * 1024,
  MAX_ROWS = 10_000,
  MAX_FIELDS = 200;

-export function parseData(source: string, format: DataFormat): DataDocument {
+export function parseData(
+  source: string,
+  format: DataFormat,
+  options: ParseDataOptions = {},
+): DataDocument {
  if (source.length > MAX_CHARS)
    throw new Error("Input exceeds the 2 MiB limit.");
+  const evidence: DataEvidence = {
+    jsonNumbers:
+      format === "json" || format === "ndjson" ? scanJsonNumbers(source) : [],
+  };
+  if (options.rejectUnsafeJsonNumbers && evidence.jsonNumbers.length)
+    throw new RangeError(
+      `JSON contains ${evidence.jsonNumbers.length} number token${evidence.jsonNumbers.length === 1 ? "" : "s"} that cannot be accepted without precision risk.`,
+    );
  let root: JsonValue;
  if (format === "json")
    root = safeJsonParse(source, {
@@ -27,8 +74,11 @@ export function parseData(source: string, format: DataFormat): DataDocument {
      maxNodes: 200_000,
    });
  else if (format === "ndjson") root = parseNdjson(source);
-  else if (format === "csv") root = parseCsvData(source);
-  else root = parseXml(source);
+  else if (format === "csv") {
+    const csv = parseCsvData(source, options);
+    root = csv.root;
+    evidence.csv = csv.evidence;
+  } else root = parseXml(source);
  const rows = normalizeRows(root);
  if (rows.length > MAX_ROWS)
    throw new Error("Dataset exceeds 10,000 query rows.");
@@ -39,29 +89,137 @@ export function parseData(source: string, format: DataFormat): DataDocument {
  ];
  if (fields.length > MAX_FIELDS)
    throw new Error("Dataset exposes more than 200 top-level fields.");
-  return { format, root, rows, fields: fields.sort() };
+  return { format, root, rows, fields: fields.sort(), evidence };
 }

-function parseCsvData(source: string): JsonValue {
+function parseCsvData(
+  source: string,
+  options: ParseDataOptions,
+): { root: JsonValue; evidence: NonNullable<DataEvidence["csv"]> } {
  const table = parseCsv(source, {
    maxRows: MAX_ROWS + 1,
    maxColumns: MAX_FIELDS,
    maxFieldChars: 100_000,
  });
-  if (!table.length) return [];
+  const inference = {
+    csvInference: options.csvInference ?? "safe",
+    inferNulls: options.inferNulls ?? false,
+    trimBeforeInference: options.trimBeforeInference ?? true,
+  };
+  if (!table.length)
+    return {
+      root: [],
+      evidence: { headers: [], rawRows: [], cells: [], inference },
+    };
  const headers = table[0]!.map(
    (value, index) => value.trim() || `column_${index + 1}`,
  );
  if (new Set(headers).size !== headers.length)
    throw new Error("CSV headers must be unique.");
-  return table
-    .slice(1)
-    .filter((row) => row.some(Boolean))
-    .map((row) =>
-      Object.fromEntries(
-        headers.map((header, index) => [header, inferScalar(row[index] ?? "")]),
-      ),
-    ) as JsonValue;
+  const rawRows = table.slice(1).filter((row) => row.some(Boolean));
+  const cells: CsvCellEvidence[] = [];
+  const root = rawRows.map((row, rowIndex) =>
+    Object.fromEntries(
+      headers.map((field, column) => {
+        const raw = row[column] ?? "";
+        const interpreted = inferCsvScalar(raw, inference);
+        cells.push({
+          row: rowIndex + 1,
+          column: column + 1,
+          field,
+          raw,
+          value: interpreted.value,
+          inferred: interpreted.kind,
+        });
+        return [field, interpreted.value];
+      }),
+    ),
+  ) as JsonValue;
+  return {
+    root,
+    evidence: {
+      headers: [...headers],
+      rawRows: rawRows.map((row) => [...row]),
+      cells,
+      inference,
+    },
+  };
+}
+
+function inferCsvScalar(
+  raw: string,
+  options: {
+    csvInference: CsvInferenceMode;
+    inferNulls: boolean;
+    trimBeforeInference: boolean;
+  },
+): { value: JsonValue; kind: CsvCellEvidence["inferred"] } {
+  if (options.csvInference === "none") return { value: raw, kind: "string" };
+  const candidate = options.trimBeforeInference ? raw.trim() : raw;
+  if (/^(?:true|false)$/iu.test(candidate))
+    return { value: candidate.toLowerCase() === "true", kind: "boolean" };
+  if (options.inferNulls && /^null$/iu.test(candidate))
+    return { value: null, kind: "null" };
+  if (/^-?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/iu.test(candidate)) {
+    const number = Number(candidate);
+    const integerToken = /^-?\d+$/u.test(candidate);
+    if (
+      Number.isFinite(number) &&
+      (options.csvInference === "aggressive" ||
+        !integerToken ||
+        Number.isSafeInteger(number))
+    )
+      return { value: number, kind: "number" };
+  }
+  return { value: raw, kind: "string" };
+}
+
+/** Identifies JSON number lexemes whose browser Number conversion loses evidence. */
+export function scanJsonNumbers(source: string): JsonNumberEvidence[] {
+  const results: JsonNumberEvidence[] = [];
+  let index = 0;
+  let inString = false;
+  while (index < source.length) {
+    const char = source[index]!;
+    if (inString) {
+      if (char === "\\") index += 2;
+      else {
+        if (char === '"') inString = false;
+        index += 1;
+      }
+      continue;
+    }
+    if (char === '"') {
+      inString = true;
+      index += 1;
+      continue;
+    }
+    const match = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/u.exec(
+      source.slice(index),
+    );
+    if (!match) {
+      index += 1;
+      continue;
+    }
+    const raw = match[0];
+    const parsed = Number(raw);
+    const integer = !/[.eE]/u.test(raw);
+    const mantissa = raw.split(/[eE]/u, 1)[0] ?? raw;
+    const significantDigits = mantissa
+      .replace(/^-|\./gu, "")
+      .replace(/^0+/u, "")
+      .replace(/0+$/u, "").length;
+    const risk = integer
+      ? Number.isSafeInteger(parsed)
+        ? undefined
+        : "unsafe-integer"
+      : !Number.isFinite(parsed) || significantDigits > 15
+        ? "precision-risk"
+        : undefined;
+    if (risk) results.push({ offset: index, raw, parsed, risk });
+    index += raw.length;
+  }
+  return results;
 }

 function parseNdjson(source: string): JsonValue {
@@ -108,34 +266,29 @@ function parseXml(source: string): JsonValue {
      const attribute = element.attributes.item(index);
      if (attribute) output[`@${attribute.name}`] = attribute.value;
    }
-    const children = [
-      ...Array.from({ length: element.childNodes.length }, (_, index) =>
-        element.childNodes.item(index),
-      ),
-    ].filter((node): node is XmlElement =>
+    const allChildren = Array.from(
+      { length: element.childNodes.length },
+      (_, index) => element.childNodes.item(index),
+    );
+    const children = allChildren.filter((node): node is XmlElement =>
      Boolean(node && node.nodeType === 1),
    );
    for (const child of children) {
-      const value = convert(child, depth + 1),
-        key = child.nodeName;
-      const existing = output[key];
-      output[key] =
+      const value = convert(child, depth + 1);
+      const existing = output[child.nodeName];
+      output[child.nodeName] =
        existing === undefined
          ? value
          : Array.isArray(existing)
            ? [...existing, value]
            : [existing, value];
    }
-    const text = [
-      ...Array.from({ length: element.childNodes.length }, (_, index) =>
-        element.childNodes.item(index),
-      ),
-    ]
+    const text = allChildren
      .filter((node) => node?.nodeType === 3 || node?.nodeType === 4)
      .map((node) => node?.nodeValue ?? "")
      .join("")
      .trim();
-    if (!Object.keys(output).length) return inferScalar(text);
+    if (!Object.keys(output).length) return text;
    if (text) output["#text"] = text;
    return output;
  };
@@ -155,14 +308,3 @@ export function normalizeRows(value: JsonValue): JsonValue[] {
 export function isObject(value: JsonValue): value is Record<string, JsonValue> {
  return Boolean(value && typeof value === "object" && !Array.isArray(value));
 }
-function inferScalar(value: string): JsonValue {
-  const trimmed = value.trim();
-  if (/^-?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/iu.test(trimmed)) {
-    const number = Number(trimmed);
-    if (Number.isFinite(number)) return number;
-  }
-  if (/^(?:true|false)$/iu.test(trimmed))
-    return trimmed.toLowerCase() === "true";
-  if (/^null$/iu.test(trimmed)) return null;
-  return value;
-}