431 lines
13 KiB
TypeScript
431 lines
13 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import {
|
|
ScanCancelledError,
|
|
analyzeLogSources,
|
|
detectLogFormat,
|
|
exportRecords,
|
|
exportLogBlob,
|
|
exportOtlpJson,
|
|
filterRecords,
|
|
parseLogLine,
|
|
redactRecords,
|
|
scanLogBlob,
|
|
scanPastedLog,
|
|
type RedactionOptions,
|
|
} from "../../src/log/model";
|
|
|
|
function streamingBlob(source: string, chunkSize = 7): Blob {
|
|
const encoded = new TextEncoder().encode(source);
|
|
return {
|
|
size: encoded.byteLength,
|
|
stream() {
|
|
let offset = 0;
|
|
return new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
if (offset >= encoded.length) {
|
|
controller.close();
|
|
return;
|
|
}
|
|
controller.enqueue(encoded.slice(offset, offset + chunkSize));
|
|
offset += chunkSize;
|
|
},
|
|
});
|
|
},
|
|
text() {
|
|
throw new Error("The scanner must not call Blob.text().");
|
|
},
|
|
} as unknown as Blob;
|
|
}
|
|
|
|
async function readBlobText(blob: Blob): Promise<string> {
|
|
return await new Promise((resolve, reject) => {
|
|
const reader = new FileReader();
|
|
reader.addEventListener("load", () => resolve(String(reader.result ?? "")));
|
|
reader.addEventListener("error", () =>
|
|
reject(reader.error ?? new Error("Could not read test Blob.")),
|
|
);
|
|
reader.readAsText(blob);
|
|
});
|
|
}
|
|
|
|
describe("incremental scanning", () => {
|
|
it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => {
|
|
const source = [
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:00:00Z",
|
|
level: "info",
|
|
service: "api",
|
|
message: "Grüße 🌍",
|
|
}),
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:01:00Z",
|
|
level: "error",
|
|
service: "api",
|
|
message: "failed",
|
|
}),
|
|
"",
|
|
].join("\n");
|
|
const progress: number[] = [];
|
|
const report = await scanLogBlob(
|
|
streamingBlob(source, 3),
|
|
{ filename: "events.jsonl", format: "auto" },
|
|
(item) => progress.push(item.bytesRead),
|
|
);
|
|
expect(report.format).toBe("jsonl");
|
|
expect(report.totalLines).toBe(2);
|
|
expect(report.preview[0]?.message).toBe("Grüße 🌍");
|
|
expect(report.levelCounts).toEqual(
|
|
expect.arrayContaining([
|
|
{ key: "info", count: 1 },
|
|
{ key: "error", count: 1 },
|
|
]),
|
|
);
|
|
expect(progress.at(-1)).toBe(report.totalBytes);
|
|
});
|
|
|
|
it("retains only the bounded preview while aggregating all lines", async () => {
|
|
const source = Array.from(
|
|
{ length: 250 },
|
|
(_, index) =>
|
|
`time=2026-09-01T10:${String(index % 60).padStart(2, "0")}:00Z level=info service=worker msg="record ${index}"`,
|
|
).join("\n");
|
|
const report = await scanLogBlob(streamingBlob(source, 113), {
|
|
format: "logfmt",
|
|
previewLimit: 100,
|
|
});
|
|
expect(report.parsedLines).toBe(250);
|
|
expect(report.preview).toHaveLength(100);
|
|
expect(report.previewDropped).toBe(150);
|
|
expect(report.timeline.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("assembles Java, .NET, and Python continuation lines with provenance", async () => {
|
|
const source = [
|
|
"2026-09-01T10:00:00Z ERROR request failed",
|
|
" at service.Handler.run(Handler.java:42)",
|
|
"Caused by: java.io.IOException: disk",
|
|
"2026-09-01T10:00:01Z INFO recovered",
|
|
"Traceback (most recent call last):",
|
|
' File "worker.py", line 4, in run',
|
|
" raise RuntimeError()",
|
|
"RuntimeError",
|
|
].join("\n");
|
|
const report = await scanLogBlob(streamingBlob(source), {
|
|
format: "plain",
|
|
multiline: "auto",
|
|
});
|
|
expect(report.totalLines).toBe(8);
|
|
expect(report.preview[0]).toMatchObject({ line: 1, lineEnd: 3 });
|
|
expect(report.preview[0]?.message).toContain("Caused by");
|
|
expect(report.preview[1]).toMatchObject({ line: 4, lineEnd: 7 });
|
|
expect(report.preview[1]?.message).toContain("worker.py");
|
|
expect(report.preview[2]).toMatchObject({ line: 8 });
|
|
});
|
|
|
|
it("supports cancellation without returning a partial replacement report", async () => {
|
|
const controller = new AbortController();
|
|
controller.abort();
|
|
await expect(
|
|
scanLogBlob(
|
|
streamingBlob("first\nsecond\n"),
|
|
{},
|
|
undefined,
|
|
controller.signal,
|
|
),
|
|
).rejects.toBeInstanceOf(ScanCancelledError);
|
|
});
|
|
|
|
it("bounds pasted input", async () => {
|
|
await expect(
|
|
scanPastedLog("x".repeat(2 * 1024 * 1024 + 1)),
|
|
).rejects.toThrow(/length/u);
|
|
});
|
|
|
|
it("truncates oversized physical lines before retaining preview text", async () => {
|
|
const report = await scanLogBlob(
|
|
streamingBlob(`${"x".repeat(256 * 1024 + 50)}\n`, 8192),
|
|
{ format: "plain" },
|
|
);
|
|
expect(report.oversizedLines).toBe(1);
|
|
expect(report.preview[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
|
|
expect(report.previewChars).toBeLessThanOrEqual(32 * 1024 * 1024);
|
|
});
|
|
});
|
|
|
|
describe("format parsers", () => {
|
|
it("detects the supported common formats", () => {
|
|
expect(detectLogFormat(['{"level":"info","message":"ok"}']).format).toBe(
|
|
"jsonl",
|
|
);
|
|
expect(
|
|
detectLogFormat([
|
|
'127.0.0.1 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 12 "-" "agent"',
|
|
]).format,
|
|
).toBe("nginx-combined");
|
|
expect(
|
|
detectLogFormat(["<34>1 2026-09-01T10:00:00Z host app 12 ID47 - event"])
|
|
.format,
|
|
).toBe("syslog");
|
|
expect(
|
|
detectLogFormat(['time=2026-09-01T10:00:00Z level=info msg="ok"']).format,
|
|
).toBe("logfmt");
|
|
});
|
|
|
|
it("extracts nginx, syslog, and logfmt fields and levels", () => {
|
|
const nginx = parseLogLine(
|
|
'192.0.2.4 - ada [01/Sep/2026:10:00:00 +0000] "GET /private HTTP/1.1" 503 18 "-" "agent"',
|
|
1,
|
|
"nginx-combined",
|
|
);
|
|
expect(nginx).toMatchObject({
|
|
valid: true,
|
|
level: "error",
|
|
source: "192.0.2.4",
|
|
});
|
|
expect(nginx.fields.status).toBe("503");
|
|
const syslog = parseLogLine(
|
|
"<34>1 2026-09-01T10:00:00Z host scheduler 42 JOB - started",
|
|
2,
|
|
"syslog",
|
|
);
|
|
expect(syslog).toMatchObject({
|
|
valid: true,
|
|
source: "scheduler",
|
|
level: "critical",
|
|
});
|
|
const logfmt = parseLogLine(
|
|
'time=2026-09-01T10:00:00Z level=warn service=queue msg="retry later"',
|
|
3,
|
|
"logfmt",
|
|
);
|
|
expect(logfmt).toMatchObject({
|
|
valid: true,
|
|
level: "warning",
|
|
source: "queue",
|
|
message: "retry later",
|
|
});
|
|
|
|
const longAgent = parseLogLine(
|
|
`192.0.2.4 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 1 "-" "${"a".repeat(5_000)}"`,
|
|
4,
|
|
"nginx-combined",
|
|
);
|
|
expect(longAgent.fields.userAgent).toHaveLength(4 * 1024);
|
|
});
|
|
|
|
it("keeps ANSI and HTML inert as plain text", () => {
|
|
const record = parseLogLine(
|
|
"\u001b[31mERROR\u001b[0m <img src=x onerror=alert(1)>",
|
|
1,
|
|
"plain",
|
|
);
|
|
expect(record.message).toBe("ERROR <img src=x onerror=alert(1)>");
|
|
});
|
|
});
|
|
|
|
describe("filter, redact, and export", () => {
|
|
const records = [
|
|
parseLogLine(
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:00:00Z",
|
|
level: "error",
|
|
service: "api",
|
|
message: "ada@example.test from 192.0.2.1",
|
|
token: "secret-value",
|
|
}),
|
|
1,
|
|
"jsonl",
|
|
),
|
|
parseLogLine(
|
|
JSON.stringify({ level: "info", service: "worker", message: "done" }),
|
|
2,
|
|
"jsonl",
|
|
),
|
|
];
|
|
const options: RedactionOptions = {
|
|
email: true,
|
|
ip: true,
|
|
uuid: true,
|
|
secrets: true,
|
|
numbers: false,
|
|
literal: "",
|
|
pseudonymize: true,
|
|
salt: "test",
|
|
};
|
|
|
|
it("filters extracted fields and produces deterministic pseudonyms", () => {
|
|
const selected = filterRecords(records, {
|
|
level: "error",
|
|
field: "service",
|
|
fieldValue: "api",
|
|
});
|
|
expect(selected).toHaveLength(1);
|
|
const first = redactRecords(selected, options);
|
|
const second = redactRecords(selected, options);
|
|
expect(first.records).toEqual(second.records);
|
|
expect(first.records[0]?.raw).not.toContain("ada@example.test");
|
|
expect(first.records[0]?.raw).not.toContain("192.0.2.1");
|
|
expect(first.counts.email).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("caps redaction expansion and reports truncated values", () => {
|
|
const expanding = parseLogLine("a".repeat(32 * 1024), 1, "plain");
|
|
const result = redactRecords([expanding], {
|
|
...options,
|
|
email: false,
|
|
ip: false,
|
|
uuid: false,
|
|
secrets: false,
|
|
literal: "a",
|
|
pseudonymize: false,
|
|
});
|
|
expect(result.records[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
|
|
expect(result.truncatedValues).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("exports normalized NDJSON, inert text, and spreadsheet-safe CSV", () => {
|
|
const dangerous = {
|
|
...records[0]!,
|
|
message: '=HYPERLINK("https://example.test")',
|
|
};
|
|
const csv = exportRecords([dangerous], "csv");
|
|
expect(csv.value).toContain("'=HYPERLINK");
|
|
expect(csv.extension).toBe("csv");
|
|
expect(
|
|
JSON.parse(exportRecords(records, "ndjson").value.split("\n")[0]!),
|
|
).toMatchObject({ line: 1, parser: "jsonl" });
|
|
expect(exportRecords(records, "text").value).toContain("ada@example.test");
|
|
});
|
|
|
|
it("performs a complete filtered and redacted second streaming pass", async () => {
|
|
const source = Array.from({ length: 250 }, (_unused, index) =>
|
|
JSON.stringify({
|
|
level: index % 2 ? "info" : "error",
|
|
service: "api",
|
|
message: `row ${index} from ada@example.test`,
|
|
}),
|
|
).join("\n");
|
|
const progress: number[] = [];
|
|
const result = await exportLogBlob(
|
|
streamingBlob(source, 31),
|
|
{
|
|
format: "ndjson",
|
|
inputFormat: "jsonl",
|
|
filters: { level: "error" },
|
|
redaction: options,
|
|
},
|
|
(value) => progress.push(value.bytesRead),
|
|
);
|
|
expect(result.rows).toBe(125);
|
|
expect(result.physicalLines).toBe(250);
|
|
expect(result.blob).toBeDefined();
|
|
const output = await readBlobText(result.blob!);
|
|
expect(output).not.toContain("ada@example.test");
|
|
expect(output.trim().split("\n")).toHaveLength(125);
|
|
expect(progress.at(-1)).toBe(source.length);
|
|
});
|
|
});
|
|
|
|
describe("multi-file temporal and trace analysis", () => {
|
|
const traceId = "0123456789abcdef0123456789abcdef";
|
|
const rootSpan = "0123456789abcdef";
|
|
const childSpan = "fedcba9876543210";
|
|
const apiRecords = [
|
|
parseLogLine(
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:00:00.000Z",
|
|
level: "info",
|
|
service: "api",
|
|
message: "start",
|
|
trace_id: traceId,
|
|
span_id: rootSpan,
|
|
}),
|
|
1,
|
|
"jsonl",
|
|
),
|
|
parseLogLine(
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T09:59:59.900Z",
|
|
level: "error",
|
|
service: "api",
|
|
message: "late arrival",
|
|
trace_id: traceId,
|
|
span_id: rootSpan,
|
|
}),
|
|
2,
|
|
"jsonl",
|
|
),
|
|
];
|
|
const workerRecords = [
|
|
parseLogLine(
|
|
JSON.stringify({
|
|
timestamp: "2026-09-01T10:00:00.250Z",
|
|
level: "info",
|
|
service: "worker",
|
|
message: "child",
|
|
traceId,
|
|
spanId: childSpan,
|
|
parentSpanId: rootSpan,
|
|
}),
|
|
1,
|
|
"jsonl",
|
|
),
|
|
];
|
|
|
|
it("merges time order and correlates trace/span evidence across files", () => {
|
|
const analysis = analyzeLogSources([
|
|
{ filename: "api.log", records: apiRecords },
|
|
{ filename: "worker.log", records: workerRecords },
|
|
]);
|
|
expect(analysis).toMatchObject({
|
|
files: 2,
|
|
inputRecords: 3,
|
|
undatedEvents: 0,
|
|
});
|
|
expect(analysis.events[0]).toMatchObject({
|
|
filename: "api.log",
|
|
line: 2,
|
|
traceId,
|
|
});
|
|
expect(analysis.groups[0]).toMatchObject({
|
|
kind: "trace",
|
|
id: traceId,
|
|
events: 3,
|
|
errors: 1,
|
|
spanCount: 2,
|
|
unresolvedParentSpanIds: [],
|
|
repeatedSpanIds: [rootSpan],
|
|
});
|
|
expect(
|
|
analysis.sourceTiming.find((item) => item.filename === "api.log")
|
|
?.backwardJumps,
|
|
).toBe(1);
|
|
});
|
|
|
|
it("maps retained records to bounded OTLP/JSON without inventing timestamps", () => {
|
|
const value = exportOtlpJson([
|
|
{ filename: "api.log", records: apiRecords },
|
|
{
|
|
filename: "undated.log",
|
|
records: [parseLogLine("INFO no clock", 1, "plain")],
|
|
},
|
|
]);
|
|
const parsed = JSON.parse(value) as {
|
|
resourceLogs: Array<{
|
|
scopeLogs: Array<{
|
|
logRecords: Array<Record<string, unknown>>;
|
|
}>;
|
|
}>;
|
|
};
|
|
const first = parsed.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!;
|
|
expect(first).toMatchObject({
|
|
traceId,
|
|
spanId: rootSpan,
|
|
severityNumber: 9,
|
|
timeUnixNano: "1788256800000000000",
|
|
});
|
|
const undated = parsed.resourceLogs[1]!.scopeLogs[0]!.logRecords[0]!;
|
|
expect(undated).not.toHaveProperty("timeUnixNano");
|
|
});
|
|
});
|