import { describe, expect, it } from "vitest";
import {
applyPipeline,
textInventory,
type TransformStep,
} from "../../src/text/pipeline";
import {
createTextArtifactEvidence,
decodeTextWithEvidence,
} from "../../src/text/evidence";
const step = (type: TransformStep["type"], option = ""): TransformStep => ({
id: type,
type,
option,
enabled: true,
});
describe("text pipelines", () => {
it("applies transformations in the declared order", () =>
expect(
applyPipeline(" b \nA\na", [
step("trim-lines"),
step("dedupe-lines"),
step("sort-lines", "en"),
]).output,
).toBe("a\nA\nb"));
it("keeps disabled steps out of the report", () =>
expect(
applyPipeline("A", [{ ...step("case", "lower"), enabled: false }]),
).toEqual({ output: "A", steps: [], warnings: [] }));
it("escapes HTML as data", () =>
expect(
applyPipeline("
", [step("escape", "html")]).output,
).toBe("<img src=x onerror=1>"));
it("strictly reverses each escape/encoding stage", () => {
const source = 'café & "quoted"\n';
for (const option of ["json", "html", "url", "base64", "hex"])
expect(
applyPipeline(applyPipeline(source, [step("escape", option)]).output, [
step("unescape", option),
]).output,
).toBe(source);
expect(() => applyPipeline("©", [step("unescape", "html")])).toThrow(
/not emitted/u,
);
expect(() => applyPipeline("w6k", [step("unescape", "base64")])).toThrow(
/canonical|padding|UTF-8/u,
);
expect(() => applyPipeline("%ZZ", [step("unescape", "url")])).toThrow();
});
it("selects columns from quoted CSV records including embedded newlines", () => {
const result = applyPipeline(
'name,note,id\r\nAlice,"hello, world",1\r\nBob,"two\nlines",2\r\n',
[step("columns", "csv|,|3,1,2")],
);
expect(result.output).toContain('1,Alice,"hello, world"');
expect(result.output).toContain('2,Bob,"two\nlines"');
expect(result.warnings[0]).toMatch(/Quoted CSV/u);
});
it("retains multi-character literal-delimiter mode", () =>
expect(
applyPipeline("left::right\na::b", [step("columns", "literal|::|2,1")])
.output,
).toBe("right::left\nb::a"));
it("reports lossy transliteration", () =>
expect(applyPipeline("Crème", [step("transliterate")]).warnings[0]).toMatch(
/lossy/u,
));
it("tracks exact line endings", () =>
expect(textInventory("a\r\nb\nc\r")).toMatchObject({
crlf: 1,
bareLf: 1,
bareCr: 1,
finalNewline: true,
}));
it("composes literal replacement, filtering, numbering and line decoration", () => {
const result = applyPipeline("old apple\nold pear\nbanana", [
step("replace-literal", JSON.stringify(["old", "fresh"])),
step("filter-lines", "fresh"),
step("prefix-lines", "- "),
step("number-lines", "10"),
step("suffix-lines", "!"),
step("join-lines", " | "),
]);
expect(result.output).toBe("10. - fresh apple! | 11. - fresh pear!");
expect(result.warnings).toEqual(
expect.arrayContaining([expect.stringMatching(/filtering removes/u)]),
);
});
it("reverses lines without reversing grapheme contents", () =>
expect(applyPipeline("😀a\nb", [step("reverse-lines")]).output).toBe(
"b\n😀a",
));
it("preflights expanding transforms before allocating their output", () => {
expect(() =>
applyPipeline("x".repeat(40), [
step("replace-literal", JSON.stringify(["x", "y".repeat(1_000_000)])),
]),
).toThrow(/32 MiB/u);
expect(() =>
applyPipeline("a\nb", [step("prefix-lines", "x".repeat(100_001))]),
).toThrow(/100,000/u);
});
});
describe("byte and portable artifact evidence", () => {
it("records BOM conflicts, UTF-8 validity and byte newline forms", () => {
const bytes = Uint8Array.of(0xef, 0xbb, 0xbf, 0x61, 0x0d, 0x0a, 0x62);
const decoded = decodeTextWithEvidence(bytes, "latin1", true);
expect(decoded.evidence).toMatchObject({
byteLength: 7,
bom: { encoding: "utf-8", bytes: 3 },
utf8: { valid: true },
byteNewlines: { crlf: 1, bareLf: 0, bareCr: 0 },
});
expect(decoded.evidence.warnings.join(" ")).toMatch(/conflicts/u);
const malformed = decodeTextWithEvidence(
Uint8Array.of(0xc3, 0x28),
"latin1",
true,
);
expect(malformed.evidence.utf8).toEqual({
valid: false,
firstInvalidOffset: 0,
});
});
it("emits exact output hashes and an honest handoff marker", async () => {
const steps = [step("case", "upper")];
const pipeline = applyPipeline("hello", steps);
const bytes = new TextEncoder().encode(pipeline.output);
const evidence = await createTextArtifactEvidence({
sourceName: "input.txt",
sourceText: "hello",
outputName: "output.txt",
outputEncoding: "utf-8",
outputBytes: bytes,
pipeline,
steps,
});
expect(evidence.output.sha256).toMatch(/^[a-f0-9]{64}$/u);
expect(evidence.handoff.supportedByThisBuild).toBe(false);
expect(evidence.pipeline.steps[0]).toMatchObject({ type: "case" });
});
});