Release Log Tools v0.1.0

2026-09-01 14:43:03 +02:00
commit 84a9978aa8
57 changed files with 10013 additions and 0 deletions
@@ -0,0 +1,262 @@
+import { describe, expect, it } from "vitest";
+import {
+  ScanCancelledError,
+  detectLogFormat,
+  exportRecords,
+  filterRecords,
+  parseLogLine,
+  redactRecords,
+  scanLogBlob,
+  scanPastedLog,
+  type RedactionOptions,
+} from "../../src/log/model";
+
+function streamingBlob(source: string, chunkSize = 7): Blob {
+  const encoded = new TextEncoder().encode(source);
+  return {
+    size: encoded.byteLength,
+    stream() {
+      let offset = 0;
+      return new ReadableStream<Uint8Array>({
+        pull(controller) {
+          if (offset >= encoded.length) {
+            controller.close();
+            return;
+          }
+          controller.enqueue(encoded.slice(offset, offset + chunkSize));
+          offset += chunkSize;
+        },
+      });
+    },
+    text() {
+      throw new Error("The scanner must not call Blob.text().");
+    },
+  } as unknown as Blob;
+}
+
+describe("incremental scanning", () => {
+  it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => {
+    const source = [
+      JSON.stringify({
+        timestamp: "2026-09-01T10:00:00Z",
+        level: "info",
+        service: "api",
+        message: "Grüße 🌍",
+      }),
+      JSON.stringify({
+        timestamp: "2026-09-01T10:01:00Z",
+        level: "error",
+        service: "api",
+        message: "failed",
+      }),
+      "",
+    ].join("\n");
+    const progress: number[] = [];
+    const report = await scanLogBlob(
+      streamingBlob(source, 3),
+      { filename: "events.jsonl", format: "auto" },
+      (item) => progress.push(item.bytesRead),
+    );
+    expect(report.format).toBe("jsonl");
+    expect(report.totalLines).toBe(2);
+    expect(report.preview[0]?.message).toBe("Grüße 🌍");
+    expect(report.levelCounts).toEqual(
+      expect.arrayContaining([
+        { key: "info", count: 1 },
+        { key: "error", count: 1 },
+      ]),
+    );
+    expect(progress.at(-1)).toBe(report.totalBytes);
+  });
+
+  it("retains only the bounded preview while aggregating all lines", async () => {
+    const source = Array.from(
+      { length: 250 },
+      (_, index) =>
+        `time=2026-09-01T10:${String(index % 60).padStart(2, "0")}:00Z level=info service=worker msg="record ${index}"`,
+    ).join("\n");
+    const report = await scanLogBlob(streamingBlob(source, 113), {
+      format: "logfmt",
+      previewLimit: 100,
+    });
+    expect(report.parsedLines).toBe(250);
+    expect(report.preview).toHaveLength(100);
+    expect(report.previewDropped).toBe(150);
+    expect(report.timeline.length).toBeGreaterThan(0);
+  });
+
+  it("supports cancellation without returning a partial replacement report", async () => {
+    const controller = new AbortController();
+    controller.abort();
+    await expect(
+      scanLogBlob(
+        streamingBlob("first\nsecond\n"),
+        {},
+        undefined,
+        controller.signal,
+      ),
+    ).rejects.toBeInstanceOf(ScanCancelledError);
+  });
+
+  it("bounds pasted input", async () => {
+    await expect(
+      scanPastedLog("x".repeat(2 * 1024 * 1024 + 1)),
+    ).rejects.toThrow(/length/u);
+  });
+
+  it("truncates oversized physical lines before retaining preview text", async () => {
+    const report = await scanLogBlob(
+      streamingBlob(`${"x".repeat(256 * 1024 + 50)}\n`, 8192),
+      { format: "plain" },
+    );
+    expect(report.oversizedLines).toBe(1);
+    expect(report.preview[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
+    expect(report.previewChars).toBeLessThanOrEqual(32 * 1024 * 1024);
+  });
+});
+
+describe("format parsers", () => {
+  it("detects the supported common formats", () => {
+    expect(detectLogFormat(['{"level":"info","message":"ok"}']).format).toBe(
+      "jsonl",
+    );
+    expect(
+      detectLogFormat([
+        '127.0.0.1 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 12 "-" "agent"',
+      ]).format,
+    ).toBe("nginx-combined");
+    expect(
+      detectLogFormat(["<34>1 2026-09-01T10:00:00Z host app 12 ID47 - event"])
+        .format,
+    ).toBe("syslog");
+    expect(
+      detectLogFormat(['time=2026-09-01T10:00:00Z level=info msg="ok"']).format,
+    ).toBe("logfmt");
+  });
+
+  it("extracts nginx, syslog, and logfmt fields and levels", () => {
+    const nginx = parseLogLine(
+      '192.0.2.4 - ada [01/Sep/2026:10:00:00 +0000] "GET /private HTTP/1.1" 503 18 "-" "agent"',
+      1,
+      "nginx-combined",
+    );
+    expect(nginx).toMatchObject({
+      valid: true,
+      level: "error",
+      source: "192.0.2.4",
+    });
+    expect(nginx.fields.status).toBe("503");
+    const syslog = parseLogLine(
+      "<34>1 2026-09-01T10:00:00Z host scheduler 42 JOB - started",
+      2,
+      "syslog",
+    );
+    expect(syslog).toMatchObject({
+      valid: true,
+      source: "scheduler",
+      level: "critical",
+    });
+    const logfmt = parseLogLine(
+      'time=2026-09-01T10:00:00Z level=warn service=queue msg="retry later"',
+      3,
+      "logfmt",
+    );
+    expect(logfmt).toMatchObject({
+      valid: true,
+      level: "warning",
+      source: "queue",
+      message: "retry later",
+    });
+
+    const longAgent = parseLogLine(
+      `192.0.2.4 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 1 "-" "${"a".repeat(5_000)}"`,
+      4,
+      "nginx-combined",
+    );
+    expect(longAgent.fields.userAgent).toHaveLength(4 * 1024);
+  });
+
+  it("keeps ANSI and HTML inert as plain text", () => {
+    const record = parseLogLine(
+      "\u001b[31mERROR\u001b[0m <img src=x onerror=alert(1)>",
+      1,
+      "plain",
+    );
+    expect(record.message).toBe("ERROR <img src=x onerror=alert(1)>");
+  });
+});
+
+describe("filter, redact, and export", () => {
+  const records = [
+    parseLogLine(
+      JSON.stringify({
+        timestamp: "2026-09-01T10:00:00Z",
+        level: "error",
+        service: "api",
+        message: "ada@example.test from 192.0.2.1",
+        token: "secret-value",
+      }),
+      1,
+      "jsonl",
+    ),
+    parseLogLine(
+      JSON.stringify({ level: "info", service: "worker", message: "done" }),
+      2,
+      "jsonl",
+    ),
+  ];
+  const options: RedactionOptions = {
+    email: true,
+    ip: true,
+    uuid: true,
+    secrets: true,
+    numbers: false,
+    literal: "",
+    pseudonymize: true,
+    salt: "test",
+  };
+
+  it("filters extracted fields and produces deterministic pseudonyms", () => {
+    const selected = filterRecords(records, {
+      level: "error",
+      field: "service",
+      fieldValue: "api",
+    });
+    expect(selected).toHaveLength(1);
+    const first = redactRecords(selected, options);
+    const second = redactRecords(selected, options);
+    expect(first.records).toEqual(second.records);
+    expect(first.records[0]?.raw).not.toContain("ada@example.test");
+    expect(first.records[0]?.raw).not.toContain("192.0.2.1");
+    expect(first.counts.email).toBeGreaterThan(0);
+  });
+
+  it("caps redaction expansion and reports truncated values", () => {
+    const expanding = parseLogLine("a".repeat(32 * 1024), 1, "plain");
+    const result = redactRecords([expanding], {
+      ...options,
+      email: false,
+      ip: false,
+      uuid: false,
+      secrets: false,
+      literal: "a",
+      pseudonymize: false,
+    });
+    expect(result.records[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
+    expect(result.truncatedValues).toBeGreaterThan(0);
+  });
+
+  it("exports normalized NDJSON, inert text, and spreadsheet-safe CSV", () => {
+    const dangerous = {
+      ...records[0]!,
+      message: '=HYPERLINK("https://example.test")',
+    };
+    const csv = exportRecords([dangerous], "csv");
+    expect(csv.value).toContain("'=HYPERLINK");
+    expect(csv.extension).toBe("csv");
+    expect(
+      JSON.parse(exportRecords(records, "ndjson").value.split("\n")[0]!),
+    ).toMatchObject({ line: 1, parser: "jsonl" });
+    expect(exportRecords(records, "text").value).toContain("ada@example.test");
+  });
+});