import { describe, expect, it } from "vitest"; import { applyPipeline, textInventory, type TransformStep, } from "../../src/text/pipeline"; import { createTextArtifactEvidence, decodeTextWithEvidence, } from "../../src/text/evidence"; const step = (type: TransformStep["type"], option = ""): TransformStep => ({ id: type, type, option, enabled: true, }); describe("text pipelines", () => { it("applies transformations in the declared order", () => expect( applyPipeline(" b \nA\na", [ step("trim-lines"), step("dedupe-lines"), step("sort-lines", "en"), ]).output, ).toBe("a\nA\nb")); it("keeps disabled steps out of the report", () => expect( applyPipeline("A", [{ ...step("case", "lower"), enabled: false }]), ).toEqual({ output: "A", steps: [], warnings: [] })); it("escapes HTML as data", () => expect( applyPipeline("", [step("escape", "html")]).output, ).toBe("<img src=x onerror=1>")); it("strictly reverses each escape/encoding stage", () => { const source = 'café & "quoted"\n'; for (const option of ["json", "html", "url", "base64", "hex"]) expect( applyPipeline(applyPipeline(source, [step("escape", option)]).output, [ step("unescape", option), ]).output, ).toBe(source); expect(() => applyPipeline("©", [step("unescape", "html")])).toThrow( /not emitted/u, ); expect(() => applyPipeline("w6k", [step("unescape", "base64")])).toThrow( /canonical|padding|UTF-8/u, ); expect(() => applyPipeline("%ZZ", [step("unescape", "url")])).toThrow(); }); it("selects columns from quoted CSV records including embedded newlines", () => { const result = applyPipeline( 'name,note,id\r\nAlice,"hello, world",1\r\nBob,"two\nlines",2\r\n', [step("columns", "csv|,|3,1,2")], ); expect(result.output).toContain('1,Alice,"hello, world"'); expect(result.output).toContain('2,Bob,"two\nlines"'); expect(result.warnings[0]).toMatch(/Quoted CSV/u); }); it("retains multi-character literal-delimiter mode", () => expect( applyPipeline("left::right\na::b", [step("columns", "literal|::|2,1")]) .output, ).toBe("right::left\nb::a")); it("reports lossy transliteration", () => expect(applyPipeline("Crème", [step("transliterate")]).warnings[0]).toMatch( /lossy/u, )); it("tracks exact line endings", () => expect(textInventory("a\r\nb\nc\r")).toMatchObject({ crlf: 1, bareLf: 1, bareCr: 1, finalNewline: true, })); it("composes literal replacement, filtering, numbering and line decoration", () => { const result = applyPipeline("old apple\nold pear\nbanana", [ step("replace-literal", JSON.stringify(["old", "fresh"])), step("filter-lines", "fresh"), step("prefix-lines", "- "), step("number-lines", "10"), step("suffix-lines", "!"), step("join-lines", " | "), ]); expect(result.output).toBe("10. - fresh apple! | 11. - fresh pear!"); expect(result.warnings).toEqual( expect.arrayContaining([expect.stringMatching(/filtering removes/u)]), ); }); it("reverses lines without reversing grapheme contents", () => expect(applyPipeline("😀a\nb", [step("reverse-lines")]).output).toBe( "b\n😀a", )); it("preflights expanding transforms before allocating their output", () => { expect(() => applyPipeline("x".repeat(40), [ step("replace-literal", JSON.stringify(["x", "y".repeat(1_000_000)])), ]), ).toThrow(/32 MiB/u); expect(() => applyPipeline("a\nb", [step("prefix-lines", "x".repeat(100_001))]), ).toThrow(/100,000/u); }); }); describe("byte and portable artifact evidence", () => { it("records BOM conflicts, UTF-8 validity and byte newline forms", () => { const bytes = Uint8Array.of(0xef, 0xbb, 0xbf, 0x61, 0x0d, 0x0a, 0x62); const decoded = decodeTextWithEvidence(bytes, "latin1", true); expect(decoded.evidence).toMatchObject({ byteLength: 7, bom: { encoding: "utf-8", bytes: 3 }, utf8: { valid: true }, byteNewlines: { crlf: 1, bareLf: 0, bareCr: 0 }, }); expect(decoded.evidence.warnings.join(" ")).toMatch(/conflicts/u); const malformed = decodeTextWithEvidence( Uint8Array.of(0xc3, 0x28), "latin1", true, ); expect(malformed.evidence.utf8).toEqual({ valid: false, firstInvalidOffset: 0, }); }); it("emits exact output hashes and an honest handoff marker", async () => { const steps = [step("case", "upper")]; const pipeline = applyPipeline("hello", steps); const bytes = new TextEncoder().encode(pipeline.output); const evidence = await createTextArtifactEvidence({ sourceName: "input.txt", sourceText: "hello", outputName: "output.txt", outputEncoding: "utf-8", outputBytes: bytes, pipeline, steps, }); expect(evidence.output.sha256).toMatch(/^[a-f0-9]{64}$/u); expect(evidence.handoff.supportedByThisBuild).toBe(false); expect(evidence.pipeline.steps[0]).toMatchObject({ type: "case" }); }); });