Release Data Tools 0.1.0
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { convertDocument } from "../../src/core/convert";
|
||||
import { DATA_LIMITS } from "../../src/core/limits";
|
||||
import { objectValue } from "../../src/core/model";
|
||||
import { parseDataDocument } from "../../src/core/parse";
|
||||
import { DataToolsError } from "../../src/core/types";
|
||||
|
||||
describe("bounded format parsers", () => {
|
||||
it("preserves JSON number lexemes exactly", () => {
|
||||
const source =
|
||||
'{"integer":9007199254740993123456789,"decimal":0.123456789012345678901}';
|
||||
const document = parseDataDocument({ source, format: "json" });
|
||||
expect(objectValue(document.root, "integer")).toMatchObject({
|
||||
type: "number",
|
||||
raw: "9007199254740993123456789",
|
||||
representation: "exact",
|
||||
});
|
||||
expect(convertDocument(document, "json").text).toContain(
|
||||
"0.123456789012345678901",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects duplicate and prototype-affecting JSON keys", () => {
|
||||
expect(() =>
|
||||
parseDataDocument({ source: '{"a":1,"a":2}', format: "json" }),
|
||||
).toThrow(DataToolsError);
|
||||
expect(() =>
|
||||
parseDataDocument({ source: '{"__proto__":1}', format: "json" }),
|
||||
).toThrow(/dangerous object key/iu);
|
||||
});
|
||||
|
||||
it("uses YAML 1.2 core semantics and rejects custom tags and merges", () => {
|
||||
const document = parseDataDocument({
|
||||
source: "answer: yes\nactual: true\ninteger: 9007199254740993\n",
|
||||
format: "yaml",
|
||||
});
|
||||
expect(objectValue(document.root, "answer")).toEqual({
|
||||
type: "string",
|
||||
value: "yes",
|
||||
});
|
||||
expect(objectValue(document.root, "integer")).toMatchObject({
|
||||
type: "number",
|
||||
raw: "9007199254740993",
|
||||
});
|
||||
expect(() =>
|
||||
parseDataDocument({ source: "value: !custom test", format: "yaml" }),
|
||||
).toThrow(/tag/iu);
|
||||
expect(() =>
|
||||
parseDataDocument({
|
||||
source: "base: &base\n a: 1\ncopy:\n <<: *base\n",
|
||||
format: "yaml",
|
||||
}),
|
||||
).toThrow(/merge/iu);
|
||||
});
|
||||
|
||||
it("keeps TOML integers exact and models date/time values", () => {
|
||||
const document = parseDataDocument({
|
||||
source:
|
||||
'title = "example"\nlarge = 9007199254740993\ncreated = 2026-09-01T10:15:30Z\n',
|
||||
format: "toml",
|
||||
});
|
||||
expect(objectValue(document.root, "large")).toMatchObject({
|
||||
type: "number",
|
||||
raw: "9007199254740993",
|
||||
representation: "bigint",
|
||||
});
|
||||
expect(objectValue(document.root, "created")).toMatchObject({
|
||||
type: "date",
|
||||
dateKind: "offset-date-time",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects XML declarations with active resolution semantics", () => {
|
||||
expect(() =>
|
||||
parseDataDocument({
|
||||
source: '<!DOCTYPE root [<!ENTITY x "expanded">]><root>&x;</root>',
|
||||
format: "xml",
|
||||
}),
|
||||
).toThrow(/DOCTYPE/iu);
|
||||
expect(() =>
|
||||
parseDataDocument({
|
||||
source:
|
||||
'<root xmlns:xi="http://www.w3.org/2001/XInclude"><xi:include href="file:///etc/passwd"/></root>',
|
||||
format: "xml",
|
||||
}),
|
||||
).toThrow(/XInclude/iu);
|
||||
expect(() =>
|
||||
parseDataDocument({
|
||||
source:
|
||||
'<root xmlns:load="http://www.w3.org/2001/XInclude"><load:include href="https://example.test/private"/></root>',
|
||||
format: "xml",
|
||||
}),
|
||||
).toThrow(/XInclude/iu);
|
||||
});
|
||||
|
||||
it("preserves XML namespaces, attributes and ordered children explicitly", () => {
|
||||
const document = parseDataDocument({
|
||||
source:
|
||||
'<n:root xmlns:n="urn:test" id="7">before<n:item>one</n:item><!--note-->after</n:root>',
|
||||
format: "xml",
|
||||
});
|
||||
expect(document.model).toBe("xml-explicit");
|
||||
expect(objectValue(document.root, "name")).toEqual({
|
||||
type: "string",
|
||||
value: "n:root",
|
||||
});
|
||||
expect(objectValue(document.root, "namespace")).toEqual({
|
||||
type: "string",
|
||||
value: "urn:test",
|
||||
});
|
||||
expect(convertDocument(document, "xml").text).toContain(
|
||||
"<n:item>one</n:item>",
|
||||
);
|
||||
});
|
||||
|
||||
it("parses delimited data as strings, including quoted fields", () => {
|
||||
const document = parseDataDocument({
|
||||
source: 'name,value\r\nAda,"1,200"\r\nZero,001',
|
||||
format: "csv",
|
||||
});
|
||||
expect(document.model).toBe("tabular");
|
||||
expect(document.root).toMatchObject({ type: "array" });
|
||||
expect(document.diagnostics).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "delimited.strings" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("preflights delimited dimensions before allocating the parsed table", () => {
|
||||
const row = new Array(DATA_LIMITS.maxColumns).fill("x").join(",");
|
||||
const source = new Array(
|
||||
Math.floor(DATA_LIMITS.maxCells / DATA_LIMITS.maxColumns) + 2,
|
||||
)
|
||||
.fill(row)
|
||||
.join("\n");
|
||||
expect(() => parseDataDocument({ source, format: "csv" })).toThrow(
|
||||
/Cell count/iu,
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an exact NDJSON line and ignores blank lines", () => {
|
||||
const document = parseDataDocument({
|
||||
source: '{"id":1}\n\n{"id":2}\n',
|
||||
format: "ndjson",
|
||||
});
|
||||
expect(document.root).toMatchObject({ type: "array", items: [{}, {}] });
|
||||
expect(document.diagnostics).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "ndjson.blank-lines" }),
|
||||
]),
|
||||
);
|
||||
expect(() =>
|
||||
parseDataDocument({
|
||||
source: '{"id":1}\n{"id":}\n',
|
||||
format: "ndjson",
|
||||
}),
|
||||
).toThrow(/line 2/iu);
|
||||
});
|
||||
|
||||
it("detects by content or filename and discloses a mismatch", () => {
|
||||
expect(
|
||||
parseDataDocument({
|
||||
source: '{"ok":true}\n{"ok":false}',
|
||||
format: "auto",
|
||||
}).format,
|
||||
).toBe("ndjson");
|
||||
const mismatch = parseDataDocument({
|
||||
source: '{"looks":"json"}',
|
||||
format: "auto",
|
||||
filename: "values.yaml",
|
||||
});
|
||||
expect(mismatch.format).toBe("yaml");
|
||||
expect(mismatch.diagnostics).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "format.possible-mismatch" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects inputs deeper than the structural cap before modelling", () => {
|
||||
const source = `${"[".repeat(DATA_LIMITS.maxDepth + 1)}0${"]".repeat(DATA_LIMITS.maxDepth + 1)}`;
|
||||
expect(() => parseDataDocument({ source, format: "json" })).toThrow(
|
||||
/depth/iu,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { convertDocument } from "../../src/core/convert";
|
||||
import { DATA_LIMITS } from "../../src/core/limits";
|
||||
import { parseDataDocument } from "../../src/core/parse";
|
||||
import {
|
||||
flattenRows,
|
||||
queryPointer,
|
||||
querySafePath,
|
||||
tableFromNode,
|
||||
} from "../../src/core/query";
|
||||
|
||||
const document = parseDataDocument({
|
||||
source:
|
||||
'{"a/b":{"~key":7},"people":[{"name":"Ada","role":"math"},{"name":"Grace","role":"code"}]}',
|
||||
format: "json",
|
||||
});
|
||||
|
||||
describe("safe data projections", () => {
|
||||
it("resolves RFC 6901 escaping", () => {
|
||||
expect(queryPointer(document.root, "/a~1b/~0key")[0]?.node).toMatchObject({
|
||||
type: "number",
|
||||
raw: "7",
|
||||
});
|
||||
});
|
||||
|
||||
it("supports a deliberately small non-evaluating path syntax", () => {
|
||||
expect(querySafePath(document.root, "$.people[*].name")).toHaveLength(2);
|
||||
expect(
|
||||
querySafePath(document.root, "$['a/b']['~key']")[0]?.node,
|
||||
).toMatchObject({ raw: "7" });
|
||||
expect(() => querySafePath(document.root, "$..name")).toThrow(
|
||||
/unsupported path syntax/iu,
|
||||
);
|
||||
expect(() => querySafePath(document.root, "$.people[?(@.role)]")).toThrow(
|
||||
/filters/iu,
|
||||
);
|
||||
expect(() =>
|
||||
queryPointer(
|
||||
document.root,
|
||||
`/${"a".repeat(DATA_LIMITS.maxQueryCharacters)}`,
|
||||
),
|
||||
).toThrow(/Query character count/iu);
|
||||
});
|
||||
|
||||
it("projects record arrays and flattened scalar leaves", () => {
|
||||
const people = queryPointer(document.root, "/people")[0]?.node;
|
||||
expect(people && tableFromNode(people)).toMatchObject({
|
||||
columns: ["name", "role"],
|
||||
rows: [
|
||||
["Ada", "math"],
|
||||
["Grace", "code"],
|
||||
],
|
||||
});
|
||||
expect(flattenRows(document.root).rows).toEqual(
|
||||
expect.arrayContaining([["/a~1b/~0key", "number", "7"]]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("explicit conversions", () => {
|
||||
it("neutralises spreadsheet formula cells only when requested", () => {
|
||||
const csv = parseDataDocument({
|
||||
source: 'name,value\ncalc,"=2+2"\nsafe,text',
|
||||
format: "csv",
|
||||
});
|
||||
const protectedOutput = convertDocument(csv, "csv", {
|
||||
spreadsheetSafe: true,
|
||||
});
|
||||
expect(protectedOutput.text).toContain("'=2+2");
|
||||
expect(protectedOutput.events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "csv.formula-neutralized" }),
|
||||
]),
|
||||
);
|
||||
const raw = convertDocument(csv, "csv", { spreadsheetSafe: false });
|
||||
expect(raw.text).toContain("=2+2");
|
||||
expect(raw.events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "csv.formula-unchecked" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("turns unsafe decimals into strings rather than silently rounding", () => {
|
||||
const precise = parseDataDocument({
|
||||
source: '{"value":0.1234567890123456789012345}',
|
||||
format: "json",
|
||||
});
|
||||
const yaml = convertDocument(precise, "yaml");
|
||||
expect(yaml.events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "yaml.unsafe-number" }),
|
||||
]),
|
||||
);
|
||||
expect(yaml.text).toContain("0.1234567890123456789012345");
|
||||
});
|
||||
|
||||
it("reports null and root coercion when targeting TOML", () => {
|
||||
const value = parseDataDocument({ source: "null", format: "json" });
|
||||
const result = convertDocument(value, "toml");
|
||||
expect(result.text).toContain('value = ""');
|
||||
expect(result.events.map((event) => event.code)).toEqual(
|
||||
expect.arrayContaining(["toml.null", "toml.root-table"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("converts ordinary data to XML using a disclosed stable convention", () => {
|
||||
const result = convertDocument(document, "xml");
|
||||
expect(result.text).toContain("<people>");
|
||||
expect(result.text).toContain("<name>Ada</name>");
|
||||
expect(result.events).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ code: "xml.mapping" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user