Release query Tools v0.1.0

2026-09-01 14:10:51 +02:00
commit 2730ce08b2
59 changed files with 8775 additions and 0 deletions
@@ -0,0 +1,168 @@
+import {
+  parseCsv,
+  safeJsonParse,
+  type JsonValue,
+} from "@add-ideas/toolbox-helpers";
+import { DOMParser, type Element as XmlElement } from "@xmldom/xmldom";
+
+export type DataFormat = "json" | "csv" | "ndjson" | "xml";
+export interface DataDocument {
+  format: DataFormat;
+  root: JsonValue;
+  rows: JsonValue[];
+  fields: string[];
+}
+const MAX_CHARS = 2 * 1024 * 1024,
+  MAX_ROWS = 10_000,
+  MAX_FIELDS = 200;
+
+export function parseData(source: string, format: DataFormat): DataDocument {
+  if (source.length > MAX_CHARS)
+    throw new Error("Input exceeds the 2 MiB limit.");
+  let root: JsonValue;
+  if (format === "json")
+    root = safeJsonParse(source, {
+      maxTextChars: MAX_CHARS,
+      maxDepth: 32,
+      maxNodes: 200_000,
+    });
+  else if (format === "ndjson") root = parseNdjson(source);
+  else if (format === "csv") root = parseCsvData(source);
+  else root = parseXml(source);
+  const rows = normalizeRows(root);
+  if (rows.length > MAX_ROWS)
+    throw new Error("Dataset exceeds 10,000 query rows.");
+  const fields = [
+    ...new Set(
+      rows.flatMap((row) => (isObject(row) ? Object.keys(row) : ["value"])),
+    ),
+  ];
+  if (fields.length > MAX_FIELDS)
+    throw new Error("Dataset exposes more than 200 top-level fields.");
+  return { format, root, rows, fields: fields.sort() };
+}
+
+function parseCsvData(source: string): JsonValue {
+  const table = parseCsv(source, {
+    maxRows: MAX_ROWS + 1,
+    maxColumns: MAX_FIELDS,
+    maxFieldChars: 100_000,
+  });
+  if (!table.length) return [];
+  const headers = table[0]!.map(
+    (value, index) => value.trim() || `column_${index + 1}`,
+  );
+  if (new Set(headers).size !== headers.length)
+    throw new Error("CSV headers must be unique.");
+  return table
+    .slice(1)
+    .filter((row) => row.some(Boolean))
+    .map((row) =>
+      Object.fromEntries(
+        headers.map((header, index) => [header, inferScalar(row[index] ?? "")]),
+      ),
+    ) as JsonValue;
+}
+
+function parseNdjson(source: string): JsonValue {
+  const lines = source.split(/\r?\n/u).filter((line) => line.trim());
+  if (lines.length > MAX_ROWS)
+    throw new Error("NDJSON exceeds 10,000 records.");
+  return lines.map((line, index) => {
+    try {
+      return safeJsonParse(line, {
+        maxTextChars: 200_000,
+        maxDepth: 32,
+        maxNodes: 20_000,
+      });
+    } catch (error) {
+      throw new Error(
+        `NDJSON line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
+      );
+    }
+  });
+}
+
+function parseXml(source: string): JsonValue {
+  if (/<!\s*(?:DOCTYPE|ENTITY)\b/iu.test(source))
+    throw new Error("XML DTD and entity declarations are not accepted.");
+  const errors: string[] = [];
+  const document = new DOMParser({
+    onError(level, message) {
+      if (level !== "warning") errors.push(message);
+    },
+  }).parseFromString(source, "application/xml");
+  if (
+    errors.length ||
+    document.getElementsByTagName("parsererror").length ||
+    !document.documentElement
+  )
+    throw new Error(`Malformed XML${errors[0] ? `: ${errors[0]}` : "."}`);
+  let nodes = 0;
+  const convert = (element: XmlElement, depth: number): JsonValue => {
+    if (++nodes > 100_000 || depth > 32)
+      throw new Error("XML structure exceeds node/depth limits.");
+    const output: Record<string, JsonValue> = {};
+    for (let index = 0; index < element.attributes.length; index += 1) {
+      const attribute = element.attributes.item(index);
+      if (attribute) output[`@${attribute.name}`] = attribute.value;
+    }
+    const children = [
+      ...Array.from({ length: element.childNodes.length }, (_, index) =>
+        element.childNodes.item(index),
+      ),
+    ].filter((node): node is XmlElement =>
+      Boolean(node && node.nodeType === 1),
+    );
+    for (const child of children) {
+      const value = convert(child, depth + 1),
+        key = child.nodeName;
+      const existing = output[key];
+      output[key] =
+        existing === undefined
+          ? value
+          : Array.isArray(existing)
+            ? [...existing, value]
+            : [existing, value];
+    }
+    const text = [
+      ...Array.from({ length: element.childNodes.length }, (_, index) =>
+        element.childNodes.item(index),
+      ),
+    ]
+      .filter((node) => node?.nodeType === 3 || node?.nodeType === 4)
+      .map((node) => node?.nodeValue ?? "")
+      .join("")
+      .trim();
+    if (!Object.keys(output).length) return inferScalar(text);
+    if (text) output["#text"] = text;
+    return output;
+  };
+  return {
+    [document.documentElement.nodeName]: convert(document.documentElement, 0),
+  };
+}
+
+export function normalizeRows(value: JsonValue): JsonValue[] {
+  if (Array.isArray(value)) return value;
+  if (isObject(value)) {
+    const arrays = Object.values(value).filter(Array.isArray);
+    if (arrays.length === 1) return arrays[0] as JsonValue[];
+  }
+  return [value];
+}
+export function isObject(value: JsonValue): value is Record<string, JsonValue> {
+  return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+function inferScalar(value: string): JsonValue {
+  const trimmed = value.trim();
+  if (/^-?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/iu.test(trimmed)) {
+    const number = Number(trimmed);
+    if (Number.isFinite(number)) return number;
+  }
+  if (/^(?:true|false)$/iu.test(trimmed))
+    return trimmed.toLowerCase() === "true";
+  if (/^null$/iu.test(trimmed)) return null;
+  return value;
+}