Release Data Tools 0.1.0

tests/browser/app.spec.ts

@@ -0,0 +1,130 @@
+import { expect, test, type Page, type Response } from "@playwright/test";
+
+const ORIGIN = "http://127.0.0.1:4173";
+
+async function localOnly(page: Page) {
+  const external: string[] = [];
+  await page.route("**/*", async (route) => {
+    const url = new URL(route.request().url());
+    if (url.origin !== ORIGIN) {
+      external.push(url.href);
+      await route.abort();
+    } else await route.continue();
+  });
+  return external;
+}
+
+function collectErrors(page: Page): string[] {
+  const errors: string[] = [];
+  page.on("pageerror", (error) => errors.push(error.message));
+  page.on("console", (message) => {
+    if (message.type() === "error") errors.push(message.text());
+  });
+  return errors;
+}
+
+test("runs its parser worker from a nested path without external requests", async ({
+  page,
+}) => {
+  const errors = collectErrors(page);
+  const external = await localOnly(page);
+  const workerResponses: Response[] = [];
+  page.on("response", (response) => {
+    if (/data\.worker-[^/]+\.js$/u.test(new URL(response.url()).pathname))
+      workerResponses.push(response);
+  });
+
+  await page.goto("/deep/nested/data/");
+  await expect(page.locator(".workbench h1")).toHaveText("Data Tools");
+  await expect(page.getByText(/Parsed as JSON/iu)).toBeVisible();
+  expect(external).toEqual([]);
+  expect(errors).toEqual([]);
+  expect(workerResponses).toHaveLength(1);
+  const worker = workerResponses[0];
+  expect(worker).toBeDefined();
+  expect(new URL(worker!.url()).pathname).toMatch(
+    /^\/deep\/nested\/data\/assets\/data\.worker-[^/]+\.js$/u,
+  );
+  expect(worker!.headers()["content-type"]).toContain("text/javascript");
+  expect(worker!.headers()["cache-control"]).toContain("immutable");
+});
+
+test("navigates tree and safe query views", async ({ page }) => {
+  await page.goto("/deep/nested/data/");
+  await expect(page.getByText(/Parsed as JSON/iu)).toBeVisible();
+  await page.getByRole("tab", { name: /Tree/iu }).click();
+  await expect(
+    page.getByRole("treeitem").filter({ hasText: "exactInteger" }),
+  ).toBeVisible();
+
+  await page.getByRole("tab", { name: /Query/iu }).click();
+  await page.getByLabel("Query syntax").selectOption("path");
+  await expect(page.getByText("4 matches")).toBeVisible();
+  await expect(page.getByText("detect", { exact: true })).toBeVisible();
+});
+
+test("opens CSV locally, projects a table and neutralises spreadsheet formulas", async ({
+  page,
+}) => {
+  await page.goto("/deep/nested/data/");
+  await page.getByTestId("data-file-input").setInputFiles({
+    name: "records.csv",
+    mimeType: "text/csv",
+    buffer: Buffer.from("name,value\nAda,001\nFormula,=2+2", "utf8"),
+  });
+  await expect(page.getByText(/Parsed as CSV/iu)).toBeVisible();
+
+  await page.getByRole("tab", { name: /Table/iu }).click();
+  await page.getByText("Use first row as headings").click();
+  const table = page.getByRole("region", { name: "Tabular data preview" });
+  await expect(table.getByRole("columnheader", { name: "name" })).toBeVisible();
+  await expect(table.getByRole("cell", { name: "001" })).toBeVisible();
+
+  await page.getByRole("tab", { name: /Convert/iu }).click();
+  await page.getByLabel("Output format").selectOption("csv");
+  await expect(page.getByTestId("conversion-output")).toHaveValue(
+    /Formula,'=2\+2/u,
+  );
+  await expect(
+    page.getByText(/Potential spreadsheet formulas/iu),
+  ).toBeVisible();
+  const downloadPromise = page.waitForEvent("download");
+  await page.getByRole("button", { name: "Download" }).click();
+  expect((await downloadPromise).suggestedFilename()).toBe("converted.csv");
+});
+
+test("shows a hardened XML error while retaining the last successful model", async ({
+  page,
+}) => {
+  await page.goto("/deep/nested/data/");
+  await expect(page.getByText(/Parsed as JSON/iu)).toBeVisible();
+  await page.getByTestId("format-select").selectOption("xml");
+  await page
+    .getByTestId("source-editor")
+    .fill('<!DOCTYPE root [<!ENTITY x "bad">]><root>&x;</root>');
+  await expect(page.getByText("xml.doctype-rejected")).toBeVisible();
+  await page.getByRole("tab", { name: /Tree/iu }).click();
+  await expect(
+    page.getByRole("treeitem").filter({ hasText: "project" }),
+  ).toBeVisible();
+});
+
+test("serves the release identity and hardened headers", async ({
+  request,
+}) => {
+  const index = await request.get("/deep/nested/data/");
+  expect(index.ok()).toBe(true);
+  expect(index.headers()["content-security-policy"]).toContain(
+    "default-src 'self'",
+  );
+  expect(index.headers()["content-security-policy"]).toContain(
+    "worker-src 'self' blob:",
+  );
+  expect(await index.text()).not.toMatch(/\b(?:src|href)=["']\//u);
+  const manifest = await request.get("/deep/nested/data/toolbox-app.json");
+  await expect(manifest.json()).resolves.toMatchObject({
+    id: "de.add-ideas.data-tools",
+    version: "0.1.0",
+    entry: "./",
+  });
+});

tests/components/workbench.test.tsx

@@ -0,0 +1,63 @@
+import { render, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it, vi } from "vitest";
+import { App } from "../../src/App";
+
+async function renderReady() {
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async () => new Response("Not found", { status: 404 })),
+  );
+  render(<App />);
+  await screen.findByText(/Parsed as JSON/iu, {}, { timeout: 2_000 });
+}
+
+describe("Data Tools workbench", () => {
+  it("renders the local shell and exact JSON summary", async () => {
+    await renderReady();
+    expect(screen.getAllByRole("heading", { name: "Data Tools" })).toHaveLength(
+      2,
+    );
+    expect(screen.getByText("Browser-local")).toBeVisible();
+    expect(screen.getByText("native")).toBeVisible();
+  });
+
+  it("keeps the previous model while a replacement is invalid", async () => {
+    await renderReady();
+    const user = userEvent.setup();
+    const editor = screen.getByTestId("source-editor");
+    await user.clear(editor);
+    await user.type(editor, '{{"broken":');
+    await screen.findByText(
+      /Unexpected|invalid|property/iu,
+      {},
+      { timeout: 2_000 },
+    );
+    await user.click(screen.getByRole("tab", { name: /Tree/iu }));
+    expect(screen.getByText("project")).toBeVisible();
+  });
+
+  it("runs safe wildcard queries", async () => {
+    await renderReady();
+    const user = userEvent.setup();
+    await user.click(screen.getByRole("tab", { name: /Query/iu }));
+    await user.selectOptions(screen.getByLabelText("Query syntax"), "path");
+    expect(screen.getByText("4 matches")).toBeVisible();
+    expect(screen.getByText("detect")).toBeVisible();
+  });
+
+  it("converts to CSV and displays disclosure", async () => {
+    await renderReady();
+    const user = userEvent.setup();
+    await user.click(screen.getByRole("tab", { name: /Convert/iu }));
+    await user.selectOptions(screen.getByLabelText("Output format"), "csv");
+    await waitFor(() =>
+      expect(
+        (screen.getByTestId("conversion-output") as HTMLTextAreaElement).value,
+      ).toContain("Key,Value"),
+    );
+    expect(
+      screen.getByRole("heading", { name: "Conversion report" }),
+    ).toBeVisible();
+  });
+});

tests/core/parse.test.ts

@@ -0,0 +1,187 @@
+import { describe, expect, it } from "vitest";
+import { convertDocument } from "../../src/core/convert";
+import { DATA_LIMITS } from "../../src/core/limits";
+import { objectValue } from "../../src/core/model";
+import { parseDataDocument } from "../../src/core/parse";
+import { DataToolsError } from "../../src/core/types";
+
+describe("bounded format parsers", () => {
+  it("preserves JSON number lexemes exactly", () => {
+    const source =
+      '{"integer":9007199254740993123456789,"decimal":0.123456789012345678901}';
+    const document = parseDataDocument({ source, format: "json" });
+    expect(objectValue(document.root, "integer")).toMatchObject({
+      type: "number",
+      raw: "9007199254740993123456789",
+      representation: "exact",
+    });
+    expect(convertDocument(document, "json").text).toContain(
+      "0.123456789012345678901",
+    );
+  });
+
+  it("rejects duplicate and prototype-affecting JSON keys", () => {
+    expect(() =>
+      parseDataDocument({ source: '{"a":1,"a":2}', format: "json" }),
+    ).toThrow(DataToolsError);
+    expect(() =>
+      parseDataDocument({ source: '{"__proto__":1}', format: "json" }),
+    ).toThrow(/dangerous object key/iu);
+  });
+
+  it("uses YAML 1.2 core semantics and rejects custom tags and merges", () => {
+    const document = parseDataDocument({
+      source: "answer: yes\nactual: true\ninteger: 9007199254740993\n",
+      format: "yaml",
+    });
+    expect(objectValue(document.root, "answer")).toEqual({
+      type: "string",
+      value: "yes",
+    });
+    expect(objectValue(document.root, "integer")).toMatchObject({
+      type: "number",
+      raw: "9007199254740993",
+    });
+    expect(() =>
+      parseDataDocument({ source: "value: !custom test", format: "yaml" }),
+    ).toThrow(/tag/iu);
+    expect(() =>
+      parseDataDocument({
+        source: "base: &base\n  a: 1\ncopy:\n  <<: *base\n",
+        format: "yaml",
+      }),
+    ).toThrow(/merge/iu);
+  });
+
+  it("keeps TOML integers exact and models date/time values", () => {
+    const document = parseDataDocument({
+      source:
+        'title = "example"\nlarge = 9007199254740993\ncreated = 2026-09-01T10:15:30Z\n',
+      format: "toml",
+    });
+    expect(objectValue(document.root, "large")).toMatchObject({
+      type: "number",
+      raw: "9007199254740993",
+      representation: "bigint",
+    });
+    expect(objectValue(document.root, "created")).toMatchObject({
+      type: "date",
+      dateKind: "offset-date-time",
+    });
+  });
+
+  it("rejects XML declarations with active resolution semantics", () => {
+    expect(() =>
+      parseDataDocument({
+        source: '<!DOCTYPE root [<!ENTITY x "expanded">]><root>&x;</root>',
+        format: "xml",
+      }),
+    ).toThrow(/DOCTYPE/iu);
+    expect(() =>
+      parseDataDocument({
+        source:
+          '<root xmlns:xi="http://www.w3.org/2001/XInclude"><xi:include href="file:///etc/passwd"/></root>',
+        format: "xml",
+      }),
+    ).toThrow(/XInclude/iu);
+    expect(() =>
+      parseDataDocument({
+        source:
+          '<root xmlns:load="http://www.w3.org/2001/XInclude"><load:include href="https://example.test/private"/></root>',
+        format: "xml",
+      }),
+    ).toThrow(/XInclude/iu);
+  });
+
+  it("preserves XML namespaces, attributes and ordered children explicitly", () => {
+    const document = parseDataDocument({
+      source:
+        '<n:root xmlns:n="urn:test" id="7">before<n:item>one</n:item><!--note-->after</n:root>',
+      format: "xml",
+    });
+    expect(document.model).toBe("xml-explicit");
+    expect(objectValue(document.root, "name")).toEqual({
+      type: "string",
+      value: "n:root",
+    });
+    expect(objectValue(document.root, "namespace")).toEqual({
+      type: "string",
+      value: "urn:test",
+    });
+    expect(convertDocument(document, "xml").text).toContain(
+      "<n:item>one</n:item>",
+    );
+  });
+
+  it("parses delimited data as strings, including quoted fields", () => {
+    const document = parseDataDocument({
+      source: 'name,value\r\nAda,"1,200"\r\nZero,001',
+      format: "csv",
+    });
+    expect(document.model).toBe("tabular");
+    expect(document.root).toMatchObject({ type: "array" });
+    expect(document.diagnostics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "delimited.strings" }),
+      ]),
+    );
+  });
+
+  it("preflights delimited dimensions before allocating the parsed table", () => {
+    const row = new Array(DATA_LIMITS.maxColumns).fill("x").join(",");
+    const source = new Array(
+      Math.floor(DATA_LIMITS.maxCells / DATA_LIMITS.maxColumns) + 2,
+    )
+      .fill(row)
+      .join("\n");
+    expect(() => parseDataDocument({ source, format: "csv" })).toThrow(
+      /Cell count/iu,
+    );
+  });
+
+  it("reports an exact NDJSON line and ignores blank lines", () => {
+    const document = parseDataDocument({
+      source: '{"id":1}\n\n{"id":2}\n',
+      format: "ndjson",
+    });
+    expect(document.root).toMatchObject({ type: "array", items: [{}, {}] });
+    expect(document.diagnostics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "ndjson.blank-lines" }),
+      ]),
+    );
+    expect(() =>
+      parseDataDocument({
+        source: '{"id":1}\n{"id":}\n',
+        format: "ndjson",
+      }),
+    ).toThrow(/line 2/iu);
+  });
+
+  it("detects by content or filename and discloses a mismatch", () => {
+    expect(
+      parseDataDocument({
+        source: '{"ok":true}\n{"ok":false}',
+        format: "auto",
+      }).format,
+    ).toBe("ndjson");
+    const mismatch = parseDataDocument({
+      source: '{"looks":"json"}',
+      format: "auto",
+      filename: "values.yaml",
+    });
+    expect(mismatch.format).toBe("yaml");
+    expect(mismatch.diagnostics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "format.possible-mismatch" }),
+      ]),
+    );
+  });
+
+  it("rejects inputs deeper than the structural cap before modelling", () => {
+    const source = `${"[".repeat(DATA_LIMITS.maxDepth + 1)}0${"]".repeat(DATA_LIMITS.maxDepth + 1)}`;
+    expect(() => parseDataDocument({ source, format: "json" })).toThrow(
+      /depth/iu,
+    );
+  });
+});

tests/core/query-convert.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, it } from "vitest";
+import { convertDocument } from "../../src/core/convert";
+import { DATA_LIMITS } from "../../src/core/limits";
+import { parseDataDocument } from "../../src/core/parse";
+import {
+  flattenRows,
+  queryPointer,
+  querySafePath,
+  tableFromNode,
+} from "../../src/core/query";
+
+const document = parseDataDocument({
+  source:
+    '{"a/b":{"~key":7},"people":[{"name":"Ada","role":"math"},{"name":"Grace","role":"code"}]}',
+  format: "json",
+});
+
+describe("safe data projections", () => {
+  it("resolves RFC 6901 escaping", () => {
+    expect(queryPointer(document.root, "/a~1b/~0key")[0]?.node).toMatchObject({
+      type: "number",
+      raw: "7",
+    });
+  });
+
+  it("supports a deliberately small non-evaluating path syntax", () => {
+    expect(querySafePath(document.root, "$.people[*].name")).toHaveLength(2);
+    expect(
+      querySafePath(document.root, "$['a/b']['~key']")[0]?.node,
+    ).toMatchObject({ raw: "7" });
+    expect(() => querySafePath(document.root, "$..name")).toThrow(
+      /unsupported path syntax/iu,
+    );
+    expect(() => querySafePath(document.root, "$.people[?(@.role)]")).toThrow(
+      /filters/iu,
+    );
+    expect(() =>
+      queryPointer(
+        document.root,
+        `/${"a".repeat(DATA_LIMITS.maxQueryCharacters)}`,
+      ),
+    ).toThrow(/Query character count/iu);
+  });
+
+  it("projects record arrays and flattened scalar leaves", () => {
+    const people = queryPointer(document.root, "/people")[0]?.node;
+    expect(people && tableFromNode(people)).toMatchObject({
+      columns: ["name", "role"],
+      rows: [
+        ["Ada", "math"],
+        ["Grace", "code"],
+      ],
+    });
+    expect(flattenRows(document.root).rows).toEqual(
+      expect.arrayContaining([["/a~1b/~0key", "number", "7"]]),
+    );
+  });
+});
+
+describe("explicit conversions", () => {
+  it("neutralises spreadsheet formula cells only when requested", () => {
+    const csv = parseDataDocument({
+      source: 'name,value\ncalc,"=2+2"\nsafe,text',
+      format: "csv",
+    });
+    const protectedOutput = convertDocument(csv, "csv", {
+      spreadsheetSafe: true,
+    });
+    expect(protectedOutput.text).toContain("'=2+2");
+    expect(protectedOutput.events).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "csv.formula-neutralized" }),
+      ]),
+    );
+    const raw = convertDocument(csv, "csv", { spreadsheetSafe: false });
+    expect(raw.text).toContain("=2+2");
+    expect(raw.events).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "csv.formula-unchecked" }),
+      ]),
+    );
+  });
+
+  it("turns unsafe decimals into strings rather than silently rounding", () => {
+    const precise = parseDataDocument({
+      source: '{"value":0.1234567890123456789012345}',
+      format: "json",
+    });
+    const yaml = convertDocument(precise, "yaml");
+    expect(yaml.events).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "yaml.unsafe-number" }),
+      ]),
+    );
+    expect(yaml.text).toContain("0.1234567890123456789012345");
+  });
+
+  it("reports null and root coercion when targeting TOML", () => {
+    const value = parseDataDocument({ source: "null", format: "json" });
+    const result = convertDocument(value, "toml");
+    expect(result.text).toContain('value = ""');
+    expect(result.events.map((event) => event.code)).toEqual(
+      expect.arrayContaining(["toml.null", "toml.root-table"]),
+    );
+  });
+
+  it("converts ordinary data to XML using a disclosed stable convention", () => {
+    const result = convertDocument(document, "xml");
+    expect(result.text).toContain("<people>");
+    expect(result.text).toContain("<name>Ada</name>");
+    expect(result.events).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ code: "xml.mapping" }),
+      ]),
+    );
+  });
+});