import { describe, expect, it } from "vitest"; import { ScanCancelledError, detectLogFormat, exportRecords, filterRecords, parseLogLine, redactRecords, scanLogBlob, scanPastedLog, type RedactionOptions, } from "../../src/log/model"; function streamingBlob(source: string, chunkSize = 7): Blob { const encoded = new TextEncoder().encode(source); return { size: encoded.byteLength, stream() { let offset = 0; return new ReadableStream({ pull(controller) { if (offset >= encoded.length) { controller.close(); return; } controller.enqueue(encoded.slice(offset, offset + chunkSize)); offset += chunkSize; }, }); }, text() { throw new Error("The scanner must not call Blob.text()."); }, } as unknown as Blob; } describe("incremental scanning", () => { it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => { const source = [ JSON.stringify({ timestamp: "2026-09-01T10:00:00Z", level: "info", service: "api", message: "Grüße 🌍", }), JSON.stringify({ timestamp: "2026-09-01T10:01:00Z", level: "error", service: "api", message: "failed", }), "", ].join("\n"); const progress: number[] = []; const report = await scanLogBlob( streamingBlob(source, 3), { filename: "events.jsonl", format: "auto" }, (item) => progress.push(item.bytesRead), ); expect(report.format).toBe("jsonl"); expect(report.totalLines).toBe(2); expect(report.preview[0]?.message).toBe("Grüße 🌍"); expect(report.levelCounts).toEqual( expect.arrayContaining([ { key: "info", count: 1 }, { key: "error", count: 1 }, ]), ); expect(progress.at(-1)).toBe(report.totalBytes); }); it("retains only the bounded preview while aggregating all lines", async () => { const source = Array.from( { length: 250 }, (_, index) => `time=2026-09-01T10:${String(index % 60).padStart(2, "0")}:00Z level=info service=worker msg="record ${index}"`, ).join("\n"); const report = await scanLogBlob(streamingBlob(source, 113), { format: "logfmt", previewLimit: 100, }); expect(report.parsedLines).toBe(250); expect(report.preview).toHaveLength(100); expect(report.previewDropped).toBe(150); expect(report.timeline.length).toBeGreaterThan(0); }); it("supports cancellation without returning a partial replacement report", async () => { const controller = new AbortController(); controller.abort(); await expect( scanLogBlob( streamingBlob("first\nsecond\n"), {}, undefined, controller.signal, ), ).rejects.toBeInstanceOf(ScanCancelledError); }); it("bounds pasted input", async () => { await expect( scanPastedLog("x".repeat(2 * 1024 * 1024 + 1)), ).rejects.toThrow(/length/u); }); it("truncates oversized physical lines before retaining preview text", async () => { const report = await scanLogBlob( streamingBlob(`${"x".repeat(256 * 1024 + 50)}\n`, 8192), { format: "plain" }, ); expect(report.oversizedLines).toBe(1); expect(report.preview[0]?.raw.length).toBeLessThanOrEqual(32 * 1024); expect(report.previewChars).toBeLessThanOrEqual(32 * 1024 * 1024); }); }); describe("format parsers", () => { it("detects the supported common formats", () => { expect(detectLogFormat(['{"level":"info","message":"ok"}']).format).toBe( "jsonl", ); expect( detectLogFormat([ '127.0.0.1 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 12 "-" "agent"', ]).format, ).toBe("nginx-combined"); expect( detectLogFormat(["<34>1 2026-09-01T10:00:00Z host app 12 ID47 - event"]) .format, ).toBe("syslog"); expect( detectLogFormat(['time=2026-09-01T10:00:00Z level=info msg="ok"']).format, ).toBe("logfmt"); }); it("extracts nginx, syslog, and logfmt fields and levels", () => { const nginx = parseLogLine( '192.0.2.4 - ada [01/Sep/2026:10:00:00 +0000] "GET /private HTTP/1.1" 503 18 "-" "agent"', 1, "nginx-combined", ); expect(nginx).toMatchObject({ valid: true, level: "error", source: "192.0.2.4", }); expect(nginx.fields.status).toBe("503"); const syslog = parseLogLine( "<34>1 2026-09-01T10:00:00Z host scheduler 42 JOB - started", 2, "syslog", ); expect(syslog).toMatchObject({ valid: true, source: "scheduler", level: "critical", }); const logfmt = parseLogLine( 'time=2026-09-01T10:00:00Z level=warn service=queue msg="retry later"', 3, "logfmt", ); expect(logfmt).toMatchObject({ valid: true, level: "warning", source: "queue", message: "retry later", }); const longAgent = parseLogLine( `192.0.2.4 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 1 "-" "${"a".repeat(5_000)}"`, 4, "nginx-combined", ); expect(longAgent.fields.userAgent).toHaveLength(4 * 1024); }); it("keeps ANSI and HTML inert as plain text", () => { const record = parseLogLine( "\u001b[31mERROR\u001b[0m ", 1, "plain", ); expect(record.message).toBe("ERROR "); }); }); describe("filter, redact, and export", () => { const records = [ parseLogLine( JSON.stringify({ timestamp: "2026-09-01T10:00:00Z", level: "error", service: "api", message: "ada@example.test from 192.0.2.1", token: "secret-value", }), 1, "jsonl", ), parseLogLine( JSON.stringify({ level: "info", service: "worker", message: "done" }), 2, "jsonl", ), ]; const options: RedactionOptions = { email: true, ip: true, uuid: true, secrets: true, numbers: false, literal: "", pseudonymize: true, salt: "test", }; it("filters extracted fields and produces deterministic pseudonyms", () => { const selected = filterRecords(records, { level: "error", field: "service", fieldValue: "api", }); expect(selected).toHaveLength(1); const first = redactRecords(selected, options); const second = redactRecords(selected, options); expect(first.records).toEqual(second.records); expect(first.records[0]?.raw).not.toContain("ada@example.test"); expect(first.records[0]?.raw).not.toContain("192.0.2.1"); expect(first.counts.email).toBeGreaterThan(0); }); it("caps redaction expansion and reports truncated values", () => { const expanding = parseLogLine("a".repeat(32 * 1024), 1, "plain"); const result = redactRecords([expanding], { ...options, email: false, ip: false, uuid: false, secrets: false, literal: "a", pseudonymize: false, }); expect(result.records[0]?.raw.length).toBeLessThanOrEqual(32 * 1024); expect(result.truncatedValues).toBeGreaterThan(0); }); it("exports normalized NDJSON, inert text, and spreadsheet-safe CSV", () => { const dangerous = { ...records[0]!, message: '=HYPERLINK("https://example.test")', }; const csv = exportRecords([dangerous], "csv"); expect(csv.value).toContain("'=HYPERLINK"); expect(csv.extension).toBe("csv"); expect( JSON.parse(exportRecords(records, "ndjson").value.split("\n")[0]!), ).toMatchObject({ line: 1, parser: "jsonl" }); expect(exportRecords(records, "text").value).toContain("ada@example.test"); }); });