263 lines
7.6 KiB
TypeScript
263 lines
7.6 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import {
|
|
ScanCancelledError,
|
|
detectLogFormat,
|
|
exportRecords,
|
|
filterRecords,
|
|
parseLogLine,
|
|
redactRecords,
|
|
scanLogBlob,
|
|
scanPastedLog,
|
|
type RedactionOptions,
|
|
} from "../../src/log/model";
|
|
|
|
function streamingBlob(source: string, chunkSize = 7): Blob {
|
|
const encoded = new TextEncoder().encode(source);
|
|
return {
|
|
size: encoded.byteLength,
|
|
stream() {
|
|
let offset = 0;
|
|
return new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
if (offset >= encoded.length) {
|
|
controller.close();
|
|
return;
|
|
}
|
|
controller.enqueue(encoded.slice(offset, offset + chunkSize));
|
|
offset += chunkSize;
|
|
},
|
|
});
|
|
},
|
|
text() {
|
|
throw new Error("The scanner must not call Blob.text().");
|
|
},
|
|
} as unknown as Blob;
|
|
}
|
|
|
|
describe("incremental scanning", () => {
|
|
it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => {
|
|
const source = [
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:00:00Z",
|
|
level: "info",
|
|
service: "api",
|
|
message: "Grüße 🌍",
|
|
}),
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:01:00Z",
|
|
level: "error",
|
|
service: "api",
|
|
message: "failed",
|
|
}),
|
|
"",
|
|
].join("\n");
|
|
const progress: number[] = [];
|
|
const report = await scanLogBlob(
|
|
streamingBlob(source, 3),
|
|
{ filename: "events.jsonl", format: "auto" },
|
|
(item) => progress.push(item.bytesRead),
|
|
);
|
|
expect(report.format).toBe("jsonl");
|
|
expect(report.totalLines).toBe(2);
|
|
expect(report.preview[0]?.message).toBe("Grüße 🌍");
|
|
expect(report.levelCounts).toEqual(
|
|
expect.arrayContaining([
|
|
{ key: "info", count: 1 },
|
|
{ key: "error", count: 1 },
|
|
]),
|
|
);
|
|
expect(progress.at(-1)).toBe(report.totalBytes);
|
|
});
|
|
|
|
it("retains only the bounded preview while aggregating all lines", async () => {
|
|
const source = Array.from(
|
|
{ length: 250 },
|
|
(_, index) =>
|
|
`time=2026-09-01T10:${String(index % 60).padStart(2, "0")}:00Z level=info service=worker msg="record ${index}"`,
|
|
).join("\n");
|
|
const report = await scanLogBlob(streamingBlob(source, 113), {
|
|
format: "logfmt",
|
|
previewLimit: 100,
|
|
});
|
|
expect(report.parsedLines).toBe(250);
|
|
expect(report.preview).toHaveLength(100);
|
|
expect(report.previewDropped).toBe(150);
|
|
expect(report.timeline.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("supports cancellation without returning a partial replacement report", async () => {
|
|
const controller = new AbortController();
|
|
controller.abort();
|
|
await expect(
|
|
scanLogBlob(
|
|
streamingBlob("first\nsecond\n"),
|
|
{},
|
|
undefined,
|
|
controller.signal,
|
|
),
|
|
).rejects.toBeInstanceOf(ScanCancelledError);
|
|
});
|
|
|
|
it("bounds pasted input", async () => {
|
|
await expect(
|
|
scanPastedLog("x".repeat(2 * 1024 * 1024 + 1)),
|
|
).rejects.toThrow(/length/u);
|
|
});
|
|
|
|
it("truncates oversized physical lines before retaining preview text", async () => {
|
|
const report = await scanLogBlob(
|
|
streamingBlob(`${"x".repeat(256 * 1024 + 50)}\n`, 8192),
|
|
{ format: "plain" },
|
|
);
|
|
expect(report.oversizedLines).toBe(1);
|
|
expect(report.preview[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
|
|
expect(report.previewChars).toBeLessThanOrEqual(32 * 1024 * 1024);
|
|
});
|
|
});
|
|
|
|
describe("format parsers", () => {
|
|
it("detects the supported common formats", () => {
|
|
expect(detectLogFormat(['{"level":"info","message":"ok"}']).format).toBe(
|
|
"jsonl",
|
|
);
|
|
expect(
|
|
detectLogFormat([
|
|
'127.0.0.1 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 12 "-" "agent"',
|
|
]).format,
|
|
).toBe("nginx-combined");
|
|
expect(
|
|
detectLogFormat(["<34>1 2026-09-01T10:00:00Z host app 12 ID47 - event"])
|
|
.format,
|
|
).toBe("syslog");
|
|
expect(
|
|
detectLogFormat(['time=2026-09-01T10:00:00Z level=info msg="ok"']).format,
|
|
).toBe("logfmt");
|
|
});
|
|
|
|
it("extracts nginx, syslog, and logfmt fields and levels", () => {
|
|
const nginx = parseLogLine(
|
|
'192.0.2.4 - ada [01/Sep/2026:10:00:00 +0000] "GET /private HTTP/1.1" 503 18 "-" "agent"',
|
|
1,
|
|
"nginx-combined",
|
|
);
|
|
expect(nginx).toMatchObject({
|
|
valid: true,
|
|
level: "error",
|
|
source: "192.0.2.4",
|
|
});
|
|
expect(nginx.fields.status).toBe("503");
|
|
const syslog = parseLogLine(
|
|
"<34>1 2026-09-01T10:00:00Z host scheduler 42 JOB - started",
|
|
2,
|
|
"syslog",
|
|
);
|
|
expect(syslog).toMatchObject({
|
|
valid: true,
|
|
source: "scheduler",
|
|
level: "critical",
|
|
});
|
|
const logfmt = parseLogLine(
|
|
'time=2026-09-01T10:00:00Z level=warn service=queue msg="retry later"',
|
|
3,
|
|
"logfmt",
|
|
);
|
|
expect(logfmt).toMatchObject({
|
|
valid: true,
|
|
level: "warning",
|
|
source: "queue",
|
|
message: "retry later",
|
|
});
|
|
|
|
const longAgent = parseLogLine(
|
|
`192.0.2.4 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 1 "-" "${"a".repeat(5_000)}"`,
|
|
4,
|
|
"nginx-combined",
|
|
);
|
|
expect(longAgent.fields.userAgent).toHaveLength(4 * 1024);
|
|
});
|
|
|
|
it("keeps ANSI and HTML inert as plain text", () => {
|
|
const record = parseLogLine(
|
|
"\u001b[31mERROR\u001b[0m <img src=x onerror=alert(1)>",
|
|
1,
|
|
"plain",
|
|
);
|
|
expect(record.message).toBe("ERROR <img src=x onerror=alert(1)>");
|
|
});
|
|
});
|
|
|
|
describe("filter, redact, and export", () => {
|
|
const records = [
|
|
parseLogLine(
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:00:00Z",
|
|
level: "error",
|
|
service: "api",
|
|
message: "ada@example.test from 192.0.2.1",
|
|
token: "secret-value",
|
|
}),
|
|
1,
|
|
"jsonl",
|
|
),
|
|
parseLogLine(
|
|
JSON.stringify({ level: "info", service: "worker", message: "done" }),
|
|
2,
|
|
"jsonl",
|
|
),
|
|
];
|
|
const options: RedactionOptions = {
|
|
email: true,
|
|
ip: true,
|
|
uuid: true,
|
|
secrets: true,
|
|
numbers: false,
|
|
literal: "",
|
|
pseudonymize: true,
|
|
salt: "test",
|
|
};
|
|
|
|
it("filters extracted fields and produces deterministic pseudonyms", () => {
|
|
const selected = filterRecords(records, {
|
|
level: "error",
|
|
field: "service",
|
|
fieldValue: "api",
|
|
});
|
|
expect(selected).toHaveLength(1);
|
|
const first = redactRecords(selected, options);
|
|
const second = redactRecords(selected, options);
|
|
expect(first.records).toEqual(second.records);
|
|
expect(first.records[0]?.raw).not.toContain("ada@example.test");
|
|
expect(first.records[0]?.raw).not.toContain("192.0.2.1");
|
|
expect(first.counts.email).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("caps redaction expansion and reports truncated values", () => {
|
|
const expanding = parseLogLine("a".repeat(32 * 1024), 1, "plain");
|
|
const result = redactRecords([expanding], {
|
|
...options,
|
|
email: false,
|
|
ip: false,
|
|
uuid: false,
|
|
secrets: false,
|
|
literal: "a",
|
|
pseudonymize: false,
|
|
});
|
|
expect(result.records[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
|
|
expect(result.truncatedValues).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("exports normalized NDJSON, inert text, and spreadsheet-safe CSV", () => {
|
|
const dangerous = {
|
|
...records[0]!,
|
|
message: '=HYPERLINK("https://example.test")',
|
|
};
|
|
const csv = exportRecords([dangerous], "csv");
|
|
expect(csv.value).toContain("'=HYPERLINK");
|
|
expect(csv.extension).toBe("csv");
|
|
expect(
|
|
JSON.parse(exportRecords(records, "ndjson").value.split("\n")[0]!),
|
|
).toMatchObject({ line: 1, parser: "jsonl" });
|
|
expect(exportRecords(records, "text").value).toContain("ada@example.test");
|
|
});
|
|
});
|