Release Text Tools 0.2.0

tests/text/pipeline.test.ts

@@ -4,6 +4,10 @@ import {
   textInventory,
   type TransformStep,
 } from "../../src/text/pipeline";
+import {
+  createTextArtifactEvidence,
+  decodeTextWithEvidence,
+} from "../../src/text/evidence";
 
 const step = (type: TransformStep["type"], option = ""): TransformStep => ({
   id: type,
@@ -28,6 +32,36 @@ describe("text pipelines", () => {
     expect(
       applyPipeline("<img src=x onerror=1>", [step("escape", "html")]).output,
     ).toBe("&lt;img src=x onerror=1&gt;"));
+  it("strictly reverses each escape/encoding stage", () => {
+    const source = 'café <x> & "quoted"\n';
+    for (const option of ["json", "html", "url", "base64", "hex"])
+      expect(
+        applyPipeline(applyPipeline(source, [step("escape", option)]).output, [
+          step("unescape", option),
+        ]).output,
+      ).toBe(source);
+    expect(() => applyPipeline("&copy;", [step("unescape", "html")])).toThrow(
+      /not emitted/u,
+    );
+    expect(() => applyPipeline("w6k", [step("unescape", "base64")])).toThrow(
+      /canonical|padding|UTF-8/u,
+    );
+    expect(() => applyPipeline("%ZZ", [step("unescape", "url")])).toThrow();
+  });
+  it("selects columns from quoted CSV records including embedded newlines", () => {
+    const result = applyPipeline(
+      'name,note,id\r\nAlice,"hello, world",1\r\nBob,"two\nlines",2\r\n',
+      [step("columns", "csv|,|3,1,2")],
+    );
+    expect(result.output).toContain('1,Alice,"hello, world"');
+    expect(result.output).toContain('2,Bob,"two\nlines"');
+    expect(result.warnings[0]).toMatch(/Quoted CSV/u);
+  });
+  it("retains multi-character literal-delimiter mode", () =>
+    expect(
+      applyPipeline("left::right\na::b", [step("columns", "literal|::|2,1")])
+        .output,
+    ).toBe("right::left\nb::a"));
   it("reports lossy transliteration", () =>
     expect(applyPipeline("Crème", [step("transliterate")]).warnings[0]).toMatch(
       /lossy/u,
@@ -39,4 +73,75 @@ describe("text pipelines", () => {
       bareCr: 1,
       finalNewline: true,
     }));
+  it("composes literal replacement, filtering, numbering and line decoration", () => {
+    const result = applyPipeline("old apple\nold pear\nbanana", [
+      step("replace-literal", JSON.stringify(["old", "fresh"])),
+      step("filter-lines", "fresh"),
+      step("prefix-lines", "- "),
+      step("number-lines", "10"),
+      step("suffix-lines", "!"),
+      step("join-lines", " | "),
+    ]);
+    expect(result.output).toBe("10. - fresh apple! | 11. - fresh pear!");
+    expect(result.warnings).toEqual(
+      expect.arrayContaining([expect.stringMatching(/filtering removes/u)]),
+    );
+  });
+
+  it("reverses lines without reversing grapheme contents", () =>
+    expect(applyPipeline("😀a\nb", [step("reverse-lines")]).output).toBe(
+      "b\n😀a",
+    ));
+
+  it("preflights expanding transforms before allocating their output", () => {
+    expect(() =>
+      applyPipeline("x".repeat(40), [
+        step("replace-literal", JSON.stringify(["x", "y".repeat(1_000_000)])),
+      ]),
+    ).toThrow(/32 MiB/u);
+    expect(() =>
+      applyPipeline("a\nb", [step("prefix-lines", "x".repeat(100_001))]),
+    ).toThrow(/100,000/u);
+  });
+});
+
+describe("byte and portable artifact evidence", () => {
+  it("records BOM conflicts, UTF-8 validity and byte newline forms", () => {
+    const bytes = Uint8Array.of(0xef, 0xbb, 0xbf, 0x61, 0x0d, 0x0a, 0x62);
+    const decoded = decodeTextWithEvidence(bytes, "latin1", true);
+    expect(decoded.evidence).toMatchObject({
+      byteLength: 7,
+      bom: { encoding: "utf-8", bytes: 3 },
+      utf8: { valid: true },
+      byteNewlines: { crlf: 1, bareLf: 0, bareCr: 0 },
+    });
+    expect(decoded.evidence.warnings.join(" ")).toMatch(/conflicts/u);
+    const malformed = decodeTextWithEvidence(
+      Uint8Array.of(0xc3, 0x28),
+      "latin1",
+      true,
+    );
+    expect(malformed.evidence.utf8).toEqual({
+      valid: false,
+      firstInvalidOffset: 0,
+    });
+  });
+
+  it("emits exact output hashes and an honest handoff marker", async () => {
+    const steps = [step("case", "upper")];
+    const pipeline = applyPipeline("hello", steps);
+    const bytes = new TextEncoder().encode(pipeline.output);
+    const evidence = await createTextArtifactEvidence({
+      sourceName: "input.txt",
+      sourceText: "hello",
+      outputName: "output.txt",
+      outputEncoding: "utf-8",
+      outputBytes: bytes,
+      pipeline,
+      steps,
+    });
+    expect(evidence.output.sha256).toMatch(/^[a-f0-9]{64}$/u);
+    expect(evidence.handoff.supportedByThisBuild).toBe(false);
+    expect(evidence.pipeline.steps[0]).toMatchObject({ type: "case" });
+  });
 });