text-tools/tests/text/pipeline.test.ts

import { describe, expect, it } from "vitest";
import {
  applyPipeline,
  textInventory,
  type TransformStep,
} from "../../src/text/pipeline";
import {
  createTextArtifactEvidence,
  decodeTextWithEvidence,
} from "../../src/text/evidence";

const step = (type: TransformStep["type"], option = ""): TransformStep => ({
  id: type,
  type,
  option,
  enabled: true,
});
describe("text pipelines", () => {
  it("applies transformations in the declared order", () =>
    expect(
      applyPipeline(" b \nA\na", [
        step("trim-lines"),
        step("dedupe-lines"),
        step("sort-lines", "en"),
      ]).output,
    ).toBe("a\nA\nb"));
  it("keeps disabled steps out of the report", () =>
    expect(
      applyPipeline("A", [{ ...step("case", "lower"), enabled: false }]),
    ).toEqual({ output: "A", steps: [], warnings: [] }));
  it("escapes HTML as data", () =>
    expect(
      applyPipeline("<img src=x onerror=1>", [step("escape", "html")]).output,
    ).toBe("&lt;img src=x onerror=1&gt;"));
  it("strictly reverses each escape/encoding stage", () => {
    const source = 'café <x> & "quoted"\n';
    for (const option of ["json", "html", "url", "base64", "hex"])
      expect(
        applyPipeline(applyPipeline(source, [step("escape", option)]).output, [
          step("unescape", option),
        ]).output,
      ).toBe(source);
    expect(() => applyPipeline("&copy;", [step("unescape", "html")])).toThrow(
      /not emitted/u,
    );
    expect(() => applyPipeline("w6k", [step("unescape", "base64")])).toThrow(
      /canonical|padding|UTF-8/u,
    );
    expect(() => applyPipeline("%ZZ", [step("unescape", "url")])).toThrow();
  });
  it("selects columns from quoted CSV records including embedded newlines", () => {
    const result = applyPipeline(
      'name,note,id\r\nAlice,"hello, world",1\r\nBob,"two\nlines",2\r\n',
      [step("columns", "csv|,|3,1,2")],
    );
    expect(result.output).toContain('1,Alice,"hello, world"');
    expect(result.output).toContain('2,Bob,"two\nlines"');
    expect(result.warnings[0]).toMatch(/Quoted CSV/u);
  });
  it("retains multi-character literal-delimiter mode", () =>
    expect(
      applyPipeline("left::right\na::b", [step("columns", "literal|::|2,1")])
        .output,
    ).toBe("right::left\nb::a"));
  it("reports lossy transliteration", () =>
    expect(applyPipeline("Crème", [step("transliterate")]).warnings[0]).toMatch(
      /lossy/u,
    ));
  it("tracks exact line endings", () =>
    expect(textInventory("a\r\nb\nc\r")).toMatchObject({
      crlf: 1,
      bareLf: 1,
      bareCr: 1,
      finalNewline: true,
    }));
  it("composes literal replacement, filtering, numbering and line decoration", () => {
    const result = applyPipeline("old apple\nold pear\nbanana", [
      step("replace-literal", JSON.stringify(["old", "fresh"])),
      step("filter-lines", "fresh"),
      step("prefix-lines", "- "),
      step("number-lines", "10"),
      step("suffix-lines", "!"),
      step("join-lines", " | "),
    ]);
    expect(result.output).toBe("10. - fresh apple! | 11. - fresh pear!");
    expect(result.warnings).toEqual(
      expect.arrayContaining([expect.stringMatching(/filtering removes/u)]),
    );
  });

  it("reverses lines without reversing grapheme contents", () =>
    expect(applyPipeline("😀a\nb", [step("reverse-lines")]).output).toBe(
      "b\n😀a",
    ));

  it("preflights expanding transforms before allocating their output", () => {
    expect(() =>
      applyPipeline("x".repeat(40), [
        step("replace-literal", JSON.stringify(["x", "y".repeat(1_000_000)])),
      ]),
    ).toThrow(/32 MiB/u);
    expect(() =>
      applyPipeline("a\nb", [step("prefix-lines", "x".repeat(100_001))]),
    ).toThrow(/100,000/u);
  });
});

describe("byte and portable artifact evidence", () => {
  it("records BOM conflicts, UTF-8 validity and byte newline forms", () => {
    const bytes = Uint8Array.of(0xef, 0xbb, 0xbf, 0x61, 0x0d, 0x0a, 0x62);
    const decoded = decodeTextWithEvidence(bytes, "latin1", true);
    expect(decoded.evidence).toMatchObject({
      byteLength: 7,
      bom: { encoding: "utf-8", bytes: 3 },
      utf8: { valid: true },
      byteNewlines: { crlf: 1, bareLf: 0, bareCr: 0 },
    });
    expect(decoded.evidence.warnings.join(" ")).toMatch(/conflicts/u);
    const malformed = decodeTextWithEvidence(
      Uint8Array.of(0xc3, 0x28),
      "latin1",
      true,
    );
    expect(malformed.evidence.utf8).toEqual({
      valid: false,
      firstInvalidOffset: 0,
    });
  });

  it("emits exact output hashes and an honest handoff marker", async () => {
    const steps = [step("case", "upper")];
    const pipeline = applyPipeline("hello", steps);
    const bytes = new TextEncoder().encode(pipeline.output);
    const evidence = await createTextArtifactEvidence({
      sourceName: "input.txt",
      sourceText: "hello",
      outputName: "output.txt",
      outputEncoding: "utf-8",
      outputBytes: bytes,
      pipeline,
      steps,
    });
    expect(evidence.output.sha256).toMatch(/^[a-f0-9]{64}$/u);
    expect(evidence.handoff.supportedByThisBuild).toBe(false);
    expect(evidence.pipeline.steps[0]).toMatchObject({ type: "case" });
  });
});