Release Log Tools v0.1.0
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
import { expect, test, type Page } from "@playwright/test";
|
||||
|
||||
const ORIGIN = "http://127.0.0.1:4182";
|
||||
async function localOnly(page: Page) {
|
||||
const external: string[] = [];
|
||||
await page.route("**/*", async (route) => {
|
||||
const url = new URL(route.request().url());
|
||||
if (url.origin !== ORIGIN) {
|
||||
external.push(url.href);
|
||||
await route.abort();
|
||||
} else await route.continue();
|
||||
});
|
||||
return external;
|
||||
}
|
||||
|
||||
test("runs from a nested path without external requests", async ({ page }) => {
|
||||
const errors: string[] = [];
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error") errors.push(message.text());
|
||||
});
|
||||
const external = await localOnly(page);
|
||||
await page.goto("/deep/nested/log/");
|
||||
await expect(
|
||||
page.getByRole("banner").getByRole("heading", { name: "Log Tools" }),
|
||||
).toBeVisible();
|
||||
await expect(page.getByText(/complete file is never loaded/u)).toBeVisible();
|
||||
expect(external).toEqual([]);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
test("streams, filters, correlates, and redacts the sample locally", async ({
|
||||
page,
|
||||
}) => {
|
||||
const external = await localOnly(page);
|
||||
await page.goto("/deep/nested/log/");
|
||||
await page.getByRole("button", { name: "Scan pasted log" }).click();
|
||||
await expect(
|
||||
page.getByText(/Completed 4 lines with JSON Lines/u),
|
||||
).toBeVisible();
|
||||
await page.getByLabel("Literal search").fill("timeout");
|
||||
await expect(page.getByText(/1 of 4 retained records match/u)).toBeVisible();
|
||||
await page.getByRole("tab", { name: "Correlation" }).click();
|
||||
await expect(
|
||||
page.getByRole("heading", { name: "Hourly timeline" }),
|
||||
).toBeVisible();
|
||||
await page.getByRole("tab", { name: "Redaction" }).click();
|
||||
await expect(page.getByLabel("Redacted preview")).not.toContainText(
|
||||
"ada@example.test",
|
||||
);
|
||||
expect(external).toEqual([]);
|
||||
});
|
||||
|
||||
test("keeps HTML-like log text inert", async ({ page }) => {
|
||||
await page.goto("/deep/nested/log/");
|
||||
await page
|
||||
.getByLabel("Pasted log")
|
||||
.fill("2026-09-01T10:00:00Z ERROR <img src=x onerror=alert(1)>");
|
||||
await page.getByRole("button", { name: "Scan pasted log" }).click();
|
||||
await expect(
|
||||
page
|
||||
.getByRole("table")
|
||||
.getByText("2026-09-01T10:00:00Z ERROR <img src=x onerror=alert(1)>", {
|
||||
exact: true,
|
||||
}),
|
||||
).toBeVisible();
|
||||
await expect(page.locator("img[src='x']")).toHaveCount(0);
|
||||
});
|
||||
|
||||
test("serves release identity and hardened headers", async ({ request }) => {
|
||||
const index = await request.get("/deep/nested/log/");
|
||||
expect(index.ok()).toBe(true);
|
||||
expect(index.headers()["content-security-policy"]).toContain(
|
||||
"connect-src 'self'",
|
||||
);
|
||||
expect(await index.text()).not.toMatch(/\b(?:src|href)=["']\//u);
|
||||
const manifest = await request.get("/deep/nested/log/toolbox-app.json");
|
||||
await expect(manifest.json()).resolves.toMatchObject({
|
||||
id: "de.add-ideas.log-tools",
|
||||
version: "0.1.0",
|
||||
entry: "./",
|
||||
privacy: { processing: "local", telemetry: false },
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { App } from "../../src/App";
|
||||
|
||||
class StreamingTestBlob {
|
||||
readonly size: number;
|
||||
private readonly bytes: Uint8Array;
|
||||
constructor(parts: readonly unknown[] = []) {
|
||||
this.bytes = new TextEncoder().encode(parts.map(String).join(""));
|
||||
this.size = this.bytes.byteLength;
|
||||
}
|
||||
stream(): ReadableStream<Uint8Array> {
|
||||
const bytes = this.bytes;
|
||||
let done = false;
|
||||
return new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (done) controller.close();
|
||||
else {
|
||||
done = true;
|
||||
controller.enqueue(bytes);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
describe("Log Tools", () => {
|
||||
it("streams the sample and exposes correlation and redaction views", async () => {
|
||||
vi.stubGlobal("Blob", StreamingTestBlob as unknown as typeof Blob);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response("Not found", { status: 404 })),
|
||||
);
|
||||
render(<App />);
|
||||
expect(
|
||||
await screen.findByRole("heading", { name: "Log Tools" }),
|
||||
).toBeVisible();
|
||||
await screen.findByRole(
|
||||
"button",
|
||||
{ name: "Scan pasted log" },
|
||||
{ timeout: 15_000 },
|
||||
);
|
||||
await userEvent.click(
|
||||
screen.getByRole("button", { name: "Scan pasted log" }),
|
||||
);
|
||||
expect(
|
||||
await screen.findByText(/Completed 4 lines with JSON Lines/u, undefined, {
|
||||
timeout: 10_000,
|
||||
}),
|
||||
).toBeVisible();
|
||||
expect(
|
||||
screen.getByText("JSON Lines", { selector: ".metrics strong" }),
|
||||
).toBeVisible();
|
||||
await userEvent.click(screen.getByRole("tab", { name: "Redaction" }));
|
||||
expect(screen.getByText("Deterministic recipes")).toBeVisible();
|
||||
expect(screen.getByLabelText("Redacted preview")).not.toHaveTextContent(
|
||||
"ada@example.test",
|
||||
);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,262 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
ScanCancelledError,
|
||||
detectLogFormat,
|
||||
exportRecords,
|
||||
filterRecords,
|
||||
parseLogLine,
|
||||
redactRecords,
|
||||
scanLogBlob,
|
||||
scanPastedLog,
|
||||
type RedactionOptions,
|
||||
} from "../../src/log/model";
|
||||
|
||||
function streamingBlob(source: string, chunkSize = 7): Blob {
|
||||
const encoded = new TextEncoder().encode(source);
|
||||
return {
|
||||
size: encoded.byteLength,
|
||||
stream() {
|
||||
let offset = 0;
|
||||
return new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (offset >= encoded.length) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(encoded.slice(offset, offset + chunkSize));
|
||||
offset += chunkSize;
|
||||
},
|
||||
});
|
||||
},
|
||||
text() {
|
||||
throw new Error("The scanner must not call Blob.text().");
|
||||
},
|
||||
} as unknown as Blob;
|
||||
}
|
||||
|
||||
describe("incremental scanning", () => {
|
||||
it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => {
|
||||
const source = [
|
||||
JSON.stringify({
|
||||
timestamp: "2026-09-01T10:00:00Z",
|
||||
level: "info",
|
||||
service: "api",
|
||||
message: "Grüße 🌍",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-09-01T10:01:00Z",
|
||||
level: "error",
|
||||
service: "api",
|
||||
message: "failed",
|
||||
}),
|
||||
"",
|
||||
].join("\n");
|
||||
const progress: number[] = [];
|
||||
const report = await scanLogBlob(
|
||||
streamingBlob(source, 3),
|
||||
{ filename: "events.jsonl", format: "auto" },
|
||||
(item) => progress.push(item.bytesRead),
|
||||
);
|
||||
expect(report.format).toBe("jsonl");
|
||||
expect(report.totalLines).toBe(2);
|
||||
expect(report.preview[0]?.message).toBe("Grüße 🌍");
|
||||
expect(report.levelCounts).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ key: "info", count: 1 },
|
||||
{ key: "error", count: 1 },
|
||||
]),
|
||||
);
|
||||
expect(progress.at(-1)).toBe(report.totalBytes);
|
||||
});
|
||||
|
||||
it("retains only the bounded preview while aggregating all lines", async () => {
|
||||
const source = Array.from(
|
||||
{ length: 250 },
|
||||
(_, index) =>
|
||||
`time=2026-09-01T10:${String(index % 60).padStart(2, "0")}:00Z level=info service=worker msg="record ${index}"`,
|
||||
).join("\n");
|
||||
const report = await scanLogBlob(streamingBlob(source, 113), {
|
||||
format: "logfmt",
|
||||
previewLimit: 100,
|
||||
});
|
||||
expect(report.parsedLines).toBe(250);
|
||||
expect(report.preview).toHaveLength(100);
|
||||
expect(report.previewDropped).toBe(150);
|
||||
expect(report.timeline.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("supports cancellation without returning a partial replacement report", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(
|
||||
scanLogBlob(
|
||||
streamingBlob("first\nsecond\n"),
|
||||
{},
|
||||
undefined,
|
||||
controller.signal,
|
||||
),
|
||||
).rejects.toBeInstanceOf(ScanCancelledError);
|
||||
});
|
||||
|
||||
it("bounds pasted input", async () => {
|
||||
await expect(
|
||||
scanPastedLog("x".repeat(2 * 1024 * 1024 + 1)),
|
||||
).rejects.toThrow(/length/u);
|
||||
});
|
||||
|
||||
it("truncates oversized physical lines before retaining preview text", async () => {
|
||||
const report = await scanLogBlob(
|
||||
streamingBlob(`${"x".repeat(256 * 1024 + 50)}\n`, 8192),
|
||||
{ format: "plain" },
|
||||
);
|
||||
expect(report.oversizedLines).toBe(1);
|
||||
expect(report.preview[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
|
||||
expect(report.previewChars).toBeLessThanOrEqual(32 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe("format parsers", () => {
|
||||
it("detects the supported common formats", () => {
|
||||
expect(detectLogFormat(['{"level":"info","message":"ok"}']).format).toBe(
|
||||
"jsonl",
|
||||
);
|
||||
expect(
|
||||
detectLogFormat([
|
||||
'127.0.0.1 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 12 "-" "agent"',
|
||||
]).format,
|
||||
).toBe("nginx-combined");
|
||||
expect(
|
||||
detectLogFormat(["<34>1 2026-09-01T10:00:00Z host app 12 ID47 - event"])
|
||||
.format,
|
||||
).toBe("syslog");
|
||||
expect(
|
||||
detectLogFormat(['time=2026-09-01T10:00:00Z level=info msg="ok"']).format,
|
||||
).toBe("logfmt");
|
||||
});
|
||||
|
||||
it("extracts nginx, syslog, and logfmt fields and levels", () => {
|
||||
const nginx = parseLogLine(
|
||||
'192.0.2.4 - ada [01/Sep/2026:10:00:00 +0000] "GET /private HTTP/1.1" 503 18 "-" "agent"',
|
||||
1,
|
||||
"nginx-combined",
|
||||
);
|
||||
expect(nginx).toMatchObject({
|
||||
valid: true,
|
||||
level: "error",
|
||||
source: "192.0.2.4",
|
||||
});
|
||||
expect(nginx.fields.status).toBe("503");
|
||||
const syslog = parseLogLine(
|
||||
"<34>1 2026-09-01T10:00:00Z host scheduler 42 JOB - started",
|
||||
2,
|
||||
"syslog",
|
||||
);
|
||||
expect(syslog).toMatchObject({
|
||||
valid: true,
|
||||
source: "scheduler",
|
||||
level: "critical",
|
||||
});
|
||||
const logfmt = parseLogLine(
|
||||
'time=2026-09-01T10:00:00Z level=warn service=queue msg="retry later"',
|
||||
3,
|
||||
"logfmt",
|
||||
);
|
||||
expect(logfmt).toMatchObject({
|
||||
valid: true,
|
||||
level: "warning",
|
||||
source: "queue",
|
||||
message: "retry later",
|
||||
});
|
||||
|
||||
const longAgent = parseLogLine(
|
||||
`192.0.2.4 - - [01/Sep/2026:10:00:00 +0000] "GET / HTTP/1.1" 200 1 "-" "${"a".repeat(5_000)}"`,
|
||||
4,
|
||||
"nginx-combined",
|
||||
);
|
||||
expect(longAgent.fields.userAgent).toHaveLength(4 * 1024);
|
||||
});
|
||||
|
||||
it("keeps ANSI and HTML inert as plain text", () => {
|
||||
const record = parseLogLine(
|
||||
"\u001b[31mERROR\u001b[0m <img src=x onerror=alert(1)>",
|
||||
1,
|
||||
"plain",
|
||||
);
|
||||
expect(record.message).toBe("ERROR <img src=x onerror=alert(1)>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("filter, redact, and export", () => {
|
||||
const records = [
|
||||
parseLogLine(
|
||||
JSON.stringify({
|
||||
timestamp: "2026-09-01T10:00:00Z",
|
||||
level: "error",
|
||||
service: "api",
|
||||
message: "ada@example.test from 192.0.2.1",
|
||||
token: "secret-value",
|
||||
}),
|
||||
1,
|
||||
"jsonl",
|
||||
),
|
||||
parseLogLine(
|
||||
JSON.stringify({ level: "info", service: "worker", message: "done" }),
|
||||
2,
|
||||
"jsonl",
|
||||
),
|
||||
];
|
||||
const options: RedactionOptions = {
|
||||
email: true,
|
||||
ip: true,
|
||||
uuid: true,
|
||||
secrets: true,
|
||||
numbers: false,
|
||||
literal: "",
|
||||
pseudonymize: true,
|
||||
salt: "test",
|
||||
};
|
||||
|
||||
it("filters extracted fields and produces deterministic pseudonyms", () => {
|
||||
const selected = filterRecords(records, {
|
||||
level: "error",
|
||||
field: "service",
|
||||
fieldValue: "api",
|
||||
});
|
||||
expect(selected).toHaveLength(1);
|
||||
const first = redactRecords(selected, options);
|
||||
const second = redactRecords(selected, options);
|
||||
expect(first.records).toEqual(second.records);
|
||||
expect(first.records[0]?.raw).not.toContain("ada@example.test");
|
||||
expect(first.records[0]?.raw).not.toContain("192.0.2.1");
|
||||
expect(first.counts.email).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("caps redaction expansion and reports truncated values", () => {
|
||||
const expanding = parseLogLine("a".repeat(32 * 1024), 1, "plain");
|
||||
const result = redactRecords([expanding], {
|
||||
...options,
|
||||
email: false,
|
||||
ip: false,
|
||||
uuid: false,
|
||||
secrets: false,
|
||||
literal: "a",
|
||||
pseudonymize: false,
|
||||
});
|
||||
expect(result.records[0]?.raw.length).toBeLessThanOrEqual(32 * 1024);
|
||||
expect(result.truncatedValues).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("exports normalized NDJSON, inert text, and spreadsheet-safe CSV", () => {
|
||||
const dangerous = {
|
||||
...records[0]!,
|
||||
message: '=HYPERLINK("https://example.test")',
|
||||
};
|
||||
const csv = exportRecords([dangerous], "csv");
|
||||
expect(csv.value).toContain("'=HYPERLINK");
|
||||
expect(csv.extension).toBe("csv");
|
||||
expect(
|
||||
JSON.parse(exportRecords(records, "ndjson").value.split("\n")[0]!),
|
||||
).toMatchObject({ line: 1, parser: "jsonl" });
|
||||
expect(exportRecords(records, "text").value).toContain("ada@example.test");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user