1 Commits
Author SHA1 Message Date
zemion f209bf78ee Release Log Tools 0.2.0
Verify / verify (push) Canceled after 0s
2026-09-02 08:33:59 +02:00
27 changed files with 1621 additions and 106 deletions
+39
View File
@@ -0,0 +1,39 @@
name: Verify
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
concurrency:
group: verify-${{ gitea.repository }}-${{ gitea.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
CI: "true"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Select declared npm version
run: npm install --global npm@11.17.0
- name: Install dependencies
run: npm ci
- name: Audit runtime dependencies
run: npm audit --omit=dev --audit-level=moderate
- name: Check, test, and build
run: npm run check
- name: Install browser engines
run: npx playwright install --with-deps chromium firefox webkit
- name: Browser tests
run: npm run test:browser
+8
View File
@@ -1,5 +1,13 @@
# Changelog # Changelog
## 0.2.0 - 2026-09-02
- Added configurable multiline assembly, multi-file temporal merge, trace/span
and request-ID correlation with bounded retained evidence.
- Added OTLP/JSON output and a complete second streaming pass for filtered,
redacted export, with direct file-system writing where available and a
bounded Blob fallback.
## 0.1.0 - 2026-09-01 ## 0.1.0 - 2026-09-01
- Added genuinely incremental `File.stream()` / `TextDecoder` scanning with progress, cancellation, line framing, and bounded memory. - Added genuinely incremental `File.stream()` / `TextDecoder` scanning with progress, cancellation, line framing, and bounded memory.
+6 -3
View File
@@ -8,13 +8,16 @@ Log Tools is a standalone local-first application in the [add·ideas Toolbox](ht
- Incremental UTF-8 file scanning with byte/line progress, cancellation, a five-million-line stop, and an eight-GiB input gate - Incremental UTF-8 file scanning with byte/line progress, cancellation, a five-million-line stop, and an eight-GiB input gate
- Bounded pasted/sample fallback and automatic or explicit plain, JSON Lines, nginx common/combined, syslog, and logfmt parsing - Bounded pasted/sample fallback and automatic or explicit plain, JSON Lines, nginx common/combined, syslog, and logfmt parsing
- Configurable streaming assembly for Java, .NET, Python, and generic indented multiline records, retaining their physical start/end lines
- Normalized level, timestamp, source, message, and common structured fields with malformed-line diagnostics - Normalized level, timestamp, source, message, and common structured fields with malformed-line diagnostics
- Whole-scan bounded level/source/hour aggregation plus a 10,000-record retained preview for literal search and field filters - Whole-scan bounded level/source/hour aggregation plus a 10,000-record retained preview for literal search and field filters
- Multi-file temporal merging for up to 16 separately streamed logs, with first-event offsets, backward-time diagnostics, W3C trace/span and common request-ID grouping, cross-file duration/source summaries, and retained-evidence caveats
- Focused OTLP/JSON log export with nanosecond timestamp strings, severity mapping, string attributes, trace/span context, and no invented timestamp for undated events
- Deterministic email, IP, UUID, credential/query-secret, long-number, and exact-literal redaction recipes with generic or salted correlation tokens - Deterministic email, IP, UUID, credential/query-secret, long-number, and exact-literal redaction recipes with generic or salted correlation tokens
- Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, with row/text caps and spreadsheet-safe CSV cells - Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, plus a complete second streaming pass with cancellation and progress; supported browsers write directly to a selected file and others use a 256 MiB Blob fallback
- Inert ANSI/HTML handling, responsive Toolbox shell themes, nested-path offline PWA support, and deterministic release archives - Inert ANSI/HTML handling, responsive Toolbox shell themes, nested-path offline PWA support, and deterministic release archives
The preview retains up to the first 10,000 nonblank parsed or malformed lines and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Search, redaction preview, and export therefore cannot include later records dropped from the preview. Auto-detection is heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, JSONL flattening is capped, and multiline stack traces remain separate plain records. The preview retains up to the first 10,000 nonblank parsed or malformed records and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Multi-file mode retains at most 3,000 records per file (50,000/128 MiB across the analysis, 20,000 merged table events), so missing parent spans mean “not present in retained logs,” not necessarily a broken trace. OTLP/JSON export is capped at 20,000 records/64 MiB of retained text. Interactive search and redaction preview operate on bounded sets, while complete single-source export reapplies the same filters and recipes during a second pass over every scanned physical line. Auto-detection and automatic multiline classification are heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, and JSONL flattening is capped.
See [Architecture](docs/ARCHITECTURE.md) and [Privacy and security](docs/PRIVACY-SECURITY.md) for exact bounds and trust assumptions. See [Architecture](docs/ARCHITECTURE.md) and [Privacy and security](docs/PRIVACY-SECURITY.md) for exact bounds and trust assumptions.
@@ -30,7 +33,7 @@ npm run test:browser
## Release ## Release
`npm run release:artifact` creates deterministic `release/log-tools-0.1.0.zip` and checksum files. `npm run release:artifact` creates deterministic `release/log-tools-0.2.0.zip` and checksum files.
## Licence ## Licence
+2 -2
View File
@@ -1,7 +1,7 @@
# Corresponding source # Corresponding source
The corresponding source for Log Tools 0.1.0 is available at: The corresponding source for Log Tools 0.2.0 is available at:
https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.1.0 https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.2.0
Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`. Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`.
+4 -4
View File
@@ -1,12 +1,12 @@
# Third-party notices # Third-party notices
Log Tools 0.1.0 directly depends on these runtime packages: Log Tools 0.2.0 directly depends on these runtime packages:
| Package | Pinned version | Declared licence | | Package | Pinned version | Declared licence |
| -------------------------------- | -------------: | ---------------- | | -------------------------------- | -------------: | ---------------- |
| `@add-ideas/toolbox-contract` | 0.2.3 | Apache-2.0 | | `@add-ideas/toolbox-contract` | 0.3.0 | Apache-2.0 |
| `@add-ideas/toolbox-helpers` | 0.1.0 | GPL-3.0-or-later | | `@add-ideas/toolbox-helpers` | 0.2.0 | GPL-3.0-or-later |
| `@add-ideas/toolbox-shell-react` | 0.2.3 | Apache-2.0 | | `@add-ideas/toolbox-shell-react` | 0.3.0 | Apache-2.0 |
| `react` | 19.2.8 | MIT | | `react` | 19.2.8 | MIT |
| `react-dom` | 19.2.8 | MIT | | `react-dom` | 19.2.8 | MIT |
+4 -2
View File
@@ -1,9 +1,11 @@
# Architecture # Architecture
Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string. Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. A bounded multiline assembler groups configured stack-trace continuations and records their first and last physical lines. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string.
Automatic detection buffers only the first 30 nonblank bounded lines, chooses among JSON Lines, nginx common/combined, RFC 5424/common RFC 3164 syslog, logfmt, and plain text, then replays that small buffer through the selected parser. Explicit selection skips detection. Parsing continues up to five million lines or eight GiB. At most the first 10,000 nonblank records and 32 MiB of retained characters are kept; individual retained raw/message strings, fields, level/source maps, hourly timestamp buckets, field names, counters, and progress are independently capped. Reads yield between progress intervals so cancellation and UI updates can run. Automatic detection buffers only the first 30 nonblank bounded lines, chooses among JSON Lines, nginx common/combined, RFC 5424/common RFC 3164 syslog, logfmt, and plain text, then replays that small buffer through the selected parser. Explicit selection skips detection. Parsing continues up to five million lines or eight GiB. At most the first 10,000 nonblank records and 32 MiB of retained characters are kept; individual retained raw/message strings, fields, level/source maps, hourly timestamp buckets, field names, counters, and progress are independently capped. Reads yield between progress intervals so cancellation and UI updates can run.
Filtering operates on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Exports use the filtered retained preview, at most 50,000 rows and 10 MiB of text. CSV prefixes spreadsheet-formula-leading cells. Filtering operates interactively on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Quick exports use the preview; `exportLogBlob()` performs a second cancellable streaming pass and reapplies parsing, filters, and redaction to all records. It writes through a supplied `WritableStream`, or returns a Blob under a 256 MiB cap. CSV prefixes spreadsheet-formula-leading cells.
Multi-file analysis streams each selected file through the same scanner, retains at most 3,000 records per file, and merges only those retained records. `analyzeLogSources()` caps file count, record count, retained characters, merged events, and correlation groups. It recognizes valid non-zero W3C-sized trace/span IDs plus bounded common request identifiers, reports unresolved parent references as retained-evidence gaps, and never adjusts timestamps. `exportOtlpJson()` produces a focused bounded OTLP/JSON request from retained records; it does not claim to reconstruct complete traces or export the unretained source.
React renders log values through text nodes only. ANSI escape sequences are removed and other control bytes are replaced; HTML-like content is never interpreted. Relative assets and a same-origin scoped service worker keep the build relocatable below nested portal routes. There is no telemetry, database, server API, worker, or remote dependency at runtime. React renders log values through text nodes only. ANSI escape sequences are removed and other control bytes are replaced; HTML-like content is never interpreted. Relative assets and a same-origin scoped service worker keep the build relocatable below nested portal routes. There is no telemetry, database, server API, worker, or remote dependency at runtime.
+22 -23
View File
@@ -1,22 +1,22 @@
{ {
"name": "log-tools", "name": "log-tools",
"version": "0.1.0", "version": "0.2.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "log-tools", "name": "log-tools",
"version": "0.1.0", "version": "0.2.0",
"license": "GPL-3.0-or-later", "license": "GPL-3.0-or-later",
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3", "@add-ideas/toolbox-contract": "0.3.0",
"@add-ideas/toolbox-helpers": "0.1.0", "@add-ideas/toolbox-helpers": "0.2.0",
"@add-ideas/toolbox-shell-react": "0.2.3", "@add-ideas/toolbox-shell-react": "0.3.0",
"react": "19.2.8", "react": "19.2.8",
"react-dom": "19.2.8" "react-dom": "19.2.8"
}, },
"devDependencies": { "devDependencies": {
"@add-ideas/toolbox-testkit": "0.2.3", "@add-ideas/toolbox-testkit": "0.3.0",
"@eslint/js": "10.0.1", "@eslint/js": "10.0.1",
"@playwright/test": "1.62.1", "@playwright/test": "1.62.1",
"@testing-library/jest-dom": "6.9.1", "@testing-library/jest-dom": "6.9.1",
@@ -42,24 +42,24 @@
} }
}, },
"node_modules/@add-ideas/toolbox-contract": { "node_modules/@add-ideas/toolbox-contract": {
"version": "0.2.3", "version": "0.3.0",
"license": "Apache-2.0", "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz",
"engines": { "integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==",
"node": ">=20" "license": "Apache-2.0"
}
}, },
"node_modules/@add-ideas/toolbox-helpers": { "node_modules/@add-ideas/toolbox-helpers": {
"version": "0.1.0", "version": "0.2.0",
"license": "GPL-3.0-or-later", "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.2.0/toolbox-helpers-0.2.0.tgz",
"engines": { "integrity": "sha512-SdOqkw+P+3J3fa5iVkzb5P15rVepB001GNV21Oh8w0CZcVL+YRltgD/s+MVcTyrNijWQf3E5vtQON/3N2LLyKg==",
"node": ">=22" "license": "GPL-3.0-or-later"
}
}, },
"node_modules/@add-ideas/toolbox-shell-react": { "node_modules/@add-ideas/toolbox-shell-react": {
"version": "0.2.3", "version": "0.3.0",
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz",
"integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==",
"license": "Apache-2.0", "license": "Apache-2.0",
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3" "@add-ideas/toolbox-contract": "0.3.0"
}, },
"peerDependencies": { "peerDependencies": {
"react": ">=18 <20", "react": ">=18 <20",
@@ -67,17 +67,16 @@
} }
}, },
"node_modules/@add-ideas/toolbox-testkit": { "node_modules/@add-ideas/toolbox-testkit": {
"version": "0.2.3", "version": "0.3.0",
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz",
"integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==",
"dev": true, "dev": true,
"license": "Apache-2.0", "license": "Apache-2.0",
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3" "@add-ideas/toolbox-contract": "0.3.0"
}, },
"bin": { "bin": {
"toolbox-check": "dist/cli.js" "toolbox-check": "dist/cli.js"
},
"engines": {
"node": ">=20"
} }
}, },
"node_modules/@adobe/css-tools": { "node_modules/@adobe/css-tools": {
+5 -5
View File
@@ -1,6 +1,6 @@
{ {
"name": "log-tools", "name": "log-tools",
"version": "0.1.0", "version": "0.2.0",
"description": "Stream, inspect, correlate, redact, and export logs locally in the browser.", "description": "Stream, inspect, correlate, redact, and export logs locally in the browser.",
"license": "GPL-3.0-or-later", "license": "GPL-3.0-or-later",
"author": "Albrecht Degering", "author": "Albrecht Degering",
@@ -39,14 +39,14 @@
"release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force" "release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force"
}, },
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3", "@add-ideas/toolbox-contract": "0.3.0",
"@add-ideas/toolbox-helpers": "0.1.0", "@add-ideas/toolbox-helpers": "0.2.0",
"@add-ideas/toolbox-shell-react": "0.2.3", "@add-ideas/toolbox-shell-react": "0.3.0",
"react": "19.2.8", "react": "19.2.8",
"react-dom": "19.2.8" "react-dom": "19.2.8"
}, },
"devDependencies": { "devDependencies": {
"@add-ideas/toolbox-testkit": "0.2.3", "@add-ideas/toolbox-testkit": "0.3.0",
"@eslint/js": "10.0.1", "@eslint/js": "10.0.1",
"@playwright/test": "1.62.1", "@playwright/test": "1.62.1",
"@testing-library/jest-dom": "6.9.1", "@testing-library/jest-dom": "6.9.1",
+20 -2
View File
@@ -15,7 +15,25 @@ export default defineConfig({
timeout: 180_000, timeout: 180_000,
}, },
projects: [ projects: [
{ name: "chromium", use: { ...devices["Desktop Chrome"] } }, {
{ name: "firefox", use: { ...devices["Desktop Firefox"] } }, name: "chromium",
testIgnore: /responsive\.spec\.ts/,
use: { ...devices["Desktop Chrome"] },
},
{
name: "firefox",
testIgnore: /responsive\.spec\.ts/,
use: { ...devices["Desktop Firefox"] },
},
{
name: "webkit",
testIgnore: /responsive\.spec\.ts/,
use: { ...devices["Desktop Safari"] },
},
{
name: "mobile-chromium",
testMatch: /responsive\.spec\.ts/,
use: { ...devices["Pixel 5"] },
},
], ],
}); });
+8
View File
@@ -1,5 +1,13 @@
# Changelog # Changelog
## 0.2.0 - 2026-09-02
- Added configurable multiline assembly, multi-file temporal merge, trace/span
and request-ID correlation with bounded retained evidence.
- Added OTLP/JSON output and a complete second streaming pass for filtered,
redacted export, with direct file-system writing where available and a
bounded Blob fallback.
## 0.1.0 - 2026-09-01 ## 0.1.0 - 2026-09-01
- Added genuinely incremental `File.stream()` / `TextDecoder` scanning with progress, cancellation, line framing, and bounded memory. - Added genuinely incremental `File.stream()` / `TextDecoder` scanning with progress, cancellation, line framing, and bounded memory.
+3 -3
View File
@@ -1,5 +1,5 @@
============================================================================== ==============================================================================
@add-ideas/toolbox-contract@0.2.3 @add-ideas/toolbox-contract@0.3.0
Declared licence: Apache-2.0 Declared licence: Apache-2.0
============================================================================== ==============================================================================
--- LICENSE --- --- LICENSE ---
@@ -198,7 +198,7 @@ Declared licence: Apache-2.0
============================================================================== ==============================================================================
@add-ideas/toolbox-helpers@0.1.0 @add-ideas/toolbox-helpers@0.2.0
Declared licence: GPL-3.0-or-later Declared licence: GPL-3.0-or-later
============================================================================== ==============================================================================
--- LICENSE --- --- LICENSE ---
@@ -879,7 +879,7 @@ Public License instead of this License. But first, please read
============================================================================== ==============================================================================
@add-ideas/toolbox-shell-react@0.2.3 @add-ideas/toolbox-shell-react@0.3.0
Declared licence: Apache-2.0 Declared licence: Apache-2.0
============================================================================== ==============================================================================
--- LICENSE --- --- LICENSE ---
+6 -3
View File
@@ -8,13 +8,16 @@ Log Tools is a standalone local-first application in the [add·ideas Toolbox](ht
- Incremental UTF-8 file scanning with byte/line progress, cancellation, a five-million-line stop, and an eight-GiB input gate - Incremental UTF-8 file scanning with byte/line progress, cancellation, a five-million-line stop, and an eight-GiB input gate
- Bounded pasted/sample fallback and automatic or explicit plain, JSON Lines, nginx common/combined, syslog, and logfmt parsing - Bounded pasted/sample fallback and automatic or explicit plain, JSON Lines, nginx common/combined, syslog, and logfmt parsing
- Configurable streaming assembly for Java, .NET, Python, and generic indented multiline records, retaining their physical start/end lines
- Normalized level, timestamp, source, message, and common structured fields with malformed-line diagnostics - Normalized level, timestamp, source, message, and common structured fields with malformed-line diagnostics
- Whole-scan bounded level/source/hour aggregation plus a 10,000-record retained preview for literal search and field filters - Whole-scan bounded level/source/hour aggregation plus a 10,000-record retained preview for literal search and field filters
- Multi-file temporal merging for up to 16 separately streamed logs, with first-event offsets, backward-time diagnostics, W3C trace/span and common request-ID grouping, cross-file duration/source summaries, and retained-evidence caveats
- Focused OTLP/JSON log export with nanosecond timestamp strings, severity mapping, string attributes, trace/span context, and no invented timestamp for undated events
- Deterministic email, IP, UUID, credential/query-secret, long-number, and exact-literal redaction recipes with generic or salted correlation tokens - Deterministic email, IP, UUID, credential/query-secret, long-number, and exact-literal redaction recipes with generic or salted correlation tokens
- Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, with row/text caps and spreadsheet-safe CSV cells - Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, plus a complete second streaming pass with cancellation and progress; supported browsers write directly to a selected file and others use a 256 MiB Blob fallback
- Inert ANSI/HTML handling, responsive Toolbox shell themes, nested-path offline PWA support, and deterministic release archives - Inert ANSI/HTML handling, responsive Toolbox shell themes, nested-path offline PWA support, and deterministic release archives
The preview retains up to the first 10,000 nonblank parsed or malformed lines and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Search, redaction preview, and export therefore cannot include later records dropped from the preview. Auto-detection is heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, JSONL flattening is capped, and multiline stack traces remain separate plain records. The preview retains up to the first 10,000 nonblank parsed or malformed records and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Multi-file mode retains at most 3,000 records per file (50,000/128 MiB across the analysis, 20,000 merged table events), so missing parent spans mean “not present in retained logs,” not necessarily a broken trace. OTLP/JSON export is capped at 20,000 records/64 MiB of retained text. Interactive search and redaction preview operate on bounded sets, while complete single-source export reapplies the same filters and recipes during a second pass over every scanned physical line. Auto-detection and automatic multiline classification are heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, and JSONL flattening is capped.
See [Architecture](docs/ARCHITECTURE.md) and [Privacy and security](docs/PRIVACY-SECURITY.md) for exact bounds and trust assumptions. See [Architecture](docs/ARCHITECTURE.md) and [Privacy and security](docs/PRIVACY-SECURITY.md) for exact bounds and trust assumptions.
@@ -30,7 +33,7 @@ npm run test:browser
## Release ## Release
`npm run release:artifact` creates deterministic `release/log-tools-0.1.0.zip` and checksum files. `npm run release:artifact` creates deterministic `release/log-tools-0.2.0.zip` and checksum files.
## Licence ## Licence
+2 -2
View File
@@ -1,7 +1,7 @@
# Corresponding source # Corresponding source
The corresponding source for Log Tools 0.1.0 is available at: The corresponding source for Log Tools 0.2.0 is available at:
https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.1.0 https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.2.0
Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`. Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`.
+4 -4
View File
@@ -1,12 +1,12 @@
# Third-party notices # Third-party notices
Log Tools 0.1.0 directly depends on these runtime packages: Log Tools 0.2.0 directly depends on these runtime packages:
| Package | Pinned version | Declared licence | | Package | Pinned version | Declared licence |
| -------------------------------- | -------------: | ---------------- | | -------------------------------- | -------------: | ---------------- |
| `@add-ideas/toolbox-contract` | 0.2.3 | Apache-2.0 | | `@add-ideas/toolbox-contract` | 0.3.0 | Apache-2.0 |
| `@add-ideas/toolbox-helpers` | 0.1.0 | GPL-3.0-or-later | | `@add-ideas/toolbox-helpers` | 0.2.0 | GPL-3.0-or-later |
| `@add-ideas/toolbox-shell-react` | 0.2.3 | Apache-2.0 | | `@add-ideas/toolbox-shell-react` | 0.3.0 | Apache-2.0 |
| `react` | 19.2.8 | MIT | | `react` | 19.2.8 | MIT |
| `react-dom` | 19.2.8 | MIT | | `react-dom` | 19.2.8 | MIT |
+4 -2
View File
@@ -1,9 +1,11 @@
# Architecture # Architecture
Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string. Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. A bounded multiline assembler groups configured stack-trace continuations and records their first and last physical lines. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string.
Automatic detection buffers only the first 30 nonblank bounded lines, chooses among JSON Lines, nginx common/combined, RFC 5424/common RFC 3164 syslog, logfmt, and plain text, then replays that small buffer through the selected parser. Explicit selection skips detection. Parsing continues up to five million lines or eight GiB. At most the first 10,000 nonblank records and 32 MiB of retained characters are kept; individual retained raw/message strings, fields, level/source maps, hourly timestamp buckets, field names, counters, and progress are independently capped. Reads yield between progress intervals so cancellation and UI updates can run. Automatic detection buffers only the first 30 nonblank bounded lines, chooses among JSON Lines, nginx common/combined, RFC 5424/common RFC 3164 syslog, logfmt, and plain text, then replays that small buffer through the selected parser. Explicit selection skips detection. Parsing continues up to five million lines or eight GiB. At most the first 10,000 nonblank records and 32 MiB of retained characters are kept; individual retained raw/message strings, fields, level/source maps, hourly timestamp buckets, field names, counters, and progress are independently capped. Reads yield between progress intervals so cancellation and UI updates can run.
Filtering operates on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Exports use the filtered retained preview, at most 50,000 rows and 10 MiB of text. CSV prefixes spreadsheet-formula-leading cells. Filtering operates interactively on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Quick exports use the preview; `exportLogBlob()` performs a second cancellable streaming pass and reapplies parsing, filters, and redaction to all records. It writes through a supplied `WritableStream`, or returns a Blob under a 256 MiB cap. CSV prefixes spreadsheet-formula-leading cells.
Multi-file analysis streams each selected file through the same scanner, retains at most 3,000 records per file, and merges only those retained records. `analyzeLogSources()` caps file count, record count, retained characters, merged events, and correlation groups. It recognizes valid non-zero W3C-sized trace/span IDs plus bounded common request identifiers, reports unresolved parent references as retained-evidence gaps, and never adjusts timestamps. `exportOtlpJson()` produces a focused bounded OTLP/JSON request from retained records; it does not claim to reconstruct complete traces or export the unretained source.
React renders log values through text nodes only. ANSI escape sequences are removed and other control bytes are replaced; HTML-like content is never interpreted. Relative assets and a same-origin scoped service worker keep the build relocatable below nested portal routes. There is no telemetry, database, server API, worker, or remote dependency at runtime. React renders log values through text nodes only. ANSI escape sequences are removed and other control bytes are replaced; HTML-like content is never interpreted. Relative assets and a same-origin scoped service worker keep the build relocatable below nested portal routes. There is no telemetry, database, server API, worker, or remote dependency at runtime.
+1 -1
View File
@@ -1,5 +1,5 @@
const CACHE_PREFIX = "log-tools-shell-"; const CACHE_PREFIX = "log-tools-shell-";
const CACHE_NAME = CACHE_PREFIX + "0.1.0"; const CACHE_NAME = CACHE_PREFIX + "0.2.0";
const CORE = ["./", "./manifest.webmanifest", "./favicon.svg"]; const CORE = ["./", "./manifest.webmanifest", "./favicon.svg"];
self.addEventListener("install", (event) => { self.addEventListener("install", (event) => {
event.waitUntil( event.waitUntil(
+18 -1
View File
@@ -3,7 +3,7 @@
"schemaVersion": 1, "schemaVersion": 1,
"id": "de.add-ideas.log-tools", "id": "de.add-ideas.log-tools",
"name": "Log Tools", "name": "Log Tools",
"version": "0.1.0", "version": "0.2.0",
"description": "Stream, inspect, correlate, redact, and export logs locally.", "description": "Stream, inspect, correlate, redact, and export logs locally.",
"entry": "./", "entry": "./",
"icon": "./favicon.svg", "icon": "./favicon.svg",
@@ -21,6 +21,23 @@
"crossOriginIsolated": false, "crossOriginIsolated": false,
"topLevelContext": false "topLevelContext": false
}, },
"io": {
"accepts": [
{ "mediaType": "text/plain", "extensions": [".log", ".txt"] },
{
"mediaType": "application/x-ndjson",
"extensions": [".ndjson", ".jsonl"]
},
{ "mediaType": "application/json", "extensions": [".json"] }
],
"produces": [
{ "mediaType": "text/plain", "extensions": [".log", ".txt"] },
{ "mediaType": "application/x-ndjson", "extensions": [".ndjson"] },
{ "mediaType": "text/csv", "extensions": [".csv"] },
{ "mediaType": "application/json", "extensions": [".json"] }
]
},
"capabilities": { "required": [], "optional": ["file-system-access"] },
"privacy": { "privacy": {
"processing": "local", "processing": "local",
"fileUploads": true, "fileUploads": true,
+6
View File
@@ -35,6 +35,12 @@ export function HelpDialog({
Stream large local log files, parse common formats, search the retained Stream large local log files, parse common formats, search the retained
preview, correlate bounded aggregates, redact identifiers, and export. preview, correlate bounded aggregates, redact identifiers, and export.
</p> </p>
<p>
Multi-file mode merges bounded per-file previews by timestamp and groups
valid trace/span or request identifiers. It reports evidence gaps rather
than assuming a complete distributed trace, and can map the retained
records to a focused OTLP/JSON payload.
</p>
<p> <p>
All processing is performed in this browser. Imported data is treated as All processing is performed in this browser. Imported data is treated as
untrusted and bounded before parsing. ANSI and HTML remain inert text, untrusted and bounded before parsing. ANSI and HTML remain inert text,
+477 -26
View File
@@ -3,14 +3,21 @@ import { triggerBlobDownload } from "@add-ideas/toolbox-helpers";
import { import {
LOG_LIMITS, LOG_LIMITS,
ScanCancelledError, ScanCancelledError,
analyzeLogSources,
exportLogBlob,
exportOtlpJson,
exportRecords, exportRecords,
filterRecords, filterRecords,
redactRecords, redactRecords,
scanLogBlob, scanLogBlob,
scanPastedLog, scanPastedLog,
type ExportResult, type ExportResult,
type FullExportProgress,
type FullExportResult,
type LogFilters, type LogFilters,
type LogFormat, type LogFormat,
type MultilineMode,
type NamedLogSource,
type RedactionOptions, type RedactionOptions,
type ScanProgress, type ScanProgress,
type ScanReport, type ScanReport,
@@ -30,7 +37,15 @@ const formatLabels: Record<LogFormat, string> = {
syslog: "Syslog (RFC 5424 / common RFC 3164)", syslog: "Syslog (RFC 5424 / common RFC 3164)",
logfmt: "logfmt", logfmt: "logfmt",
}; };
type Tab = "explore" | "correlate" | "redact" | "export"; const multilineLabels: Record<MultilineMode, string> = {
off: "One event per physical line",
auto: "Auto: Java, .NET, and Python traces",
java: "Java stack traces",
dotnet: ".NET stack traces",
python: "Python tracebacks",
indented: "Every indented continuation",
};
type Tab = "explore" | "correlate" | "traces" | "redact" | "export";
const defaultRedaction: RedactionOptions = { const defaultRedaction: RedactionOptions = {
email: true, email: true,
ip: true, ip: true,
@@ -62,7 +77,9 @@ function save(result: ExportResult) {
export function Workbench() { export function Workbench() {
const [tab, setTab] = useState<Tab>("explore"); const [tab, setTab] = useState<Tab>("explore");
const [format, setFormat] = useState<LogFormat>("auto"); const [format, setFormat] = useState<LogFormat>("auto");
const [multiline, setMultiline] = useState<MultilineMode>("auto");
const [file, setFile] = useState<File>(); const [file, setFile] = useState<File>();
const [sourceBlob, setSourceBlob] = useState<Blob>();
const [paste, setPaste] = useState(sample); const [paste, setPaste] = useState(sample);
const [report, setReport] = useState<ScanReport>(); const [report, setReport] = useState<ScanReport>();
const [progress, setProgress] = useState<ScanProgress>(); const [progress, setProgress] = useState<ScanProgress>();
@@ -79,7 +96,15 @@ export function Workbench() {
); );
const [redactExport, setRedactExport] = useState(true); const [redactExport, setRedactExport] = useState(true);
const [lastExport, setLastExport] = useState<ExportResult>(); const [lastExport, setLastExport] = useState<ExportResult>();
const [lastFullExport, setLastFullExport] = useState<FullExportResult>();
const [multiReports, setMultiReports] = useState<ScanReport[]>([]);
const [traceScanning, setTraceScanning] = useState(false);
const [traceProgress, setTraceProgress] = useState("");
const [exporting, setExporting] = useState(false);
const [exportProgress, setExportProgress] = useState<FullExportProgress>();
const controller = useRef<AbortController | undefined>(undefined); const controller = useRef<AbortController | undefined>(undefined);
const exportController = useRef<AbortController | undefined>(undefined);
const traceController = useRef<AbortController | undefined>(undefined);
const filtered = useMemo( const filtered = useMemo(
() => filterRecords(report?.preview ?? [], filters), () => filterRecords(report?.preview ?? [], filters),
@@ -91,6 +116,22 @@ export function Workbench() {
); );
const levels = report?.levelCounts.map((item) => item.key) ?? []; const levels = report?.levelCounts.map((item) => item.key) ?? [];
const sources = report?.sourceCounts.map((item) => item.key) ?? []; const sources = report?.sourceCounts.map((item) => item.key) ?? [];
const traceSources = useMemo<NamedLogSource[]>(
() =>
multiReports.length
? multiReports.map((item) => ({
filename: item.filename,
records: item.preview,
}))
: report
? [{ filename: report.filename, records: report.preview }]
: [],
[multiReports, report],
);
const traceAnalysis = useMemo(
() => analyzeLogSources(traceSources),
[traceSources],
);
const scan = async (blob: Blob, filename: string, pasted = false) => { const scan = async (blob: Blob, filename: string, pasted = false) => {
controller.current?.abort(); controller.current?.abort();
@@ -104,18 +145,19 @@ export function Workbench() {
const next = pasted const next = pasted
? await scanPastedLog( ? await scanPastedLog(
paste, paste,
{ format, filename }, { format, filename, multiline },
setProgress, setProgress,
nextController.signal, nextController.signal,
) )
: await scanLogBlob( : await scanLogBlob(
blob, blob,
{ format, filename }, { format, filename, multiline },
setProgress, setProgress,
nextController.signal, nextController.signal,
); );
if (controller.current !== nextController) return; if (controller.current !== nextController) return;
setReport(next); setReport(next);
setSourceBlob(blob);
setFilters({}); setFilters({});
setLastExport(undefined); setLastExport(undefined);
setStatus( setStatus(
@@ -147,19 +189,135 @@ export function Workbench() {
}; };
const clear = () => { const clear = () => {
controller.current?.abort(); controller.current?.abort();
exportController.current?.abort();
traceController.current?.abort();
setFile(undefined); setFile(undefined);
setPaste(""); setPaste("");
setReport(undefined); setReport(undefined);
setSourceBlob(undefined);
setProgress(undefined); setProgress(undefined);
setMultiReports([]);
setTraceProgress("");
setError(""); setError("");
setStatus("Workspace cleared from application memory."); setStatus("Workspace cleared from application memory.");
}; };
const scanTraceFiles = async (files: FileList | null) => {
if (!files?.length) return;
if (files.length > 16) {
setError("Multi-file analysis accepts at most 16 files at once.");
return;
}
traceController.current?.abort();
const nextController = new AbortController();
traceController.current = nextController;
setTraceScanning(true);
setError("");
const next: ScanReport[] = [];
try {
for (const [index, selected] of [...files].entries()) {
setTraceProgress(
`Scanning ${index + 1} of ${files.length}: ${selected.name}`,
);
const scanned = await scanLogBlob(
selected,
{
format,
multiline,
filename: selected.webkitRelativePath || selected.name,
previewLimit: 3_000,
},
(value) =>
setTraceProgress(
`${index + 1} of ${files.length}: ${selected.name} · ${bytes(value.bytesRead)} of ${bytes(value.totalBytes)}`,
),
nextController.signal,
);
next.push(scanned);
}
if (traceController.current !== nextController) return;
setMultiReports(next);
setTraceProgress(
`${next.length} files scanned; correlation uses ${next.reduce((sum, item) => sum + item.preview.length, 0).toLocaleString()} bounded preview records.`,
);
} catch (reason) {
if (traceController.current !== nextController) return;
if (reason instanceof ScanCancelledError)
setTraceProgress(
"Multi-file scan cancelled; the previous analysis remains.",
);
else setError(message(reason, "Could not scan the log set."));
} finally {
if (traceController.current === nextController) {
setTraceScanning(false);
traceController.current = undefined;
}
}
};
const prepareExport = () => { const prepareExport = () => {
const records = redactExport ? redacted.records : filtered; const records = redactExport ? redacted.records : filtered;
const result = exportRecords(records, exportFormat); const result = exportRecords(records, exportFormat);
setLastExport(result); setLastExport(result);
save(result); save(result);
}; };
const exportCompleteSource = async () => {
if (!report || !sourceBlob) return;
exportController.current?.abort();
const nextController = new AbortController();
exportController.current = nextController;
setExporting(true);
setExportProgress({
bytesRead: 0,
totalBytes: sourceBlob.size,
physicalLines: 0,
rows: 0,
bytesWritten: 0,
});
setError("");
try {
const picker = (
window as Window & {
showSaveFilePicker?: (options: { suggestedName: string }) => Promise<{
createWritable(): Promise<WritableStream<Uint8Array>>;
}>;
}
).showSaveFilePicker;
let writable: WritableStream<Uint8Array> | undefined;
if (picker) {
const handle = await picker({
suggestedName: `filtered-logs.${exportFormat === "text" ? "log" : exportFormat}`,
});
writable = await handle.createWritable();
}
const result = await exportLogBlob(
sourceBlob,
{
format: exportFormat,
inputFormat: report.format,
filters,
...(redactExport ? { redaction } : {}),
multiline,
...(writable ? { writable } : {}),
},
setExportProgress,
nextController.signal,
);
if (exportController.current !== nextController) return;
if (result.blob) {
triggerBlobDownload(result.blob, `filtered-logs.${result.extension}`);
}
setLastFullExport(result);
} catch (reason) {
if (exportController.current !== nextController) return;
if (reason instanceof DOMException && reason.name === "AbortError")
return;
setError(message(reason, "Could not export the complete source."));
} finally {
if (exportController.current === nextController) {
setExporting(false);
exportController.current = undefined;
}
}
};
const progressValue = progress?.totalBytes const progressValue = progress?.totalBytes
? Math.min(100, (progress.bytesRead / progress.totalBytes) * 100) ? Math.min(100, (progress.bytesRead / progress.totalBytes) * 100)
: 0; : 0;
@@ -205,6 +363,22 @@ export function Workbench() {
))} ))}
</select> </select>
</label> </label>
<label className="field">
<span>Multiline events</span>
<select
value={multiline}
onChange={(event) =>
setMultiline(event.target.value as MultilineMode)
}
disabled={scanning}
>
{Object.entries(multilineLabels).map(([value, label]) => (
<option value={value} key={value}>
{label}
</option>
))}
</select>
</label>
<label className="button file-button"> <label className="button file-button">
Choose and scan local log Choose and scan local log
<input <input
@@ -337,37 +511,40 @@ export function Workbench() {
<section className="panel workspace"> <section className="panel workspace">
<div <div
className="workspace-tabs" className="workspace-tabs"
role="tablist" role="group"
aria-label="Log workbench views" aria-label="Log workbench views"
> >
<button <button
type="button" type="button"
role="tab" aria-pressed={tab === "explore"}
aria-selected={tab === "explore"}
onClick={() => setTab("explore")} onClick={() => setTab("explore")}
> >
Explore Explore
</button> </button>
<button <button
type="button" type="button"
role="tab" aria-pressed={tab === "correlate"}
aria-selected={tab === "correlate"}
onClick={() => setTab("correlate")} onClick={() => setTab("correlate")}
> >
Correlation Correlation
</button> </button>
<button <button
type="button" type="button"
role="tab" aria-pressed={tab === "traces"}
aria-selected={tab === "redact"} onClick={() => setTab("traces")}
>
Multi-file traces
</button>
<button
type="button"
aria-pressed={tab === "redact"}
onClick={() => setTab("redact")} onClick={() => setTab("redact")}
> >
Redaction Redaction
</button> </button>
<button <button
type="button" type="button"
role="tab" aria-pressed={tab === "export"}
aria-selected={tab === "export"}
onClick={() => setTab("export")} onClick={() => setTab("export")}
> >
Export Export
@@ -375,7 +552,7 @@ export function Workbench() {
</div> </div>
{tab === "explore" && ( {tab === "explore" && (
<div className="stack" role="tabpanel"> <div className="stack">
<div className="filter-grid"> <div className="filter-grid">
<label className="field"> <label className="field">
<span>Literal search</span> <span>Literal search</span>
@@ -502,7 +679,7 @@ export function Workbench() {
)} )}
{tab === "correlate" && ( {tab === "correlate" && (
<div className="correlation-grid" role="tabpanel"> <div className="correlation-grid">
<section className="stack"> <section className="stack">
<h2>Levels</h2> <h2>Levels</h2>
<ol className="count-list"> <ol className="count-list">
@@ -584,8 +761,241 @@ export function Workbench() {
</div> </div>
)} )}
{tab === "traces" && (
<div className="stack">
<div className="panel-heading compact-heading">
<div>
<p className="eyebrow">Bounded retained previews</p>
<h2>Multi-file temporal and trace analysis</h2>
<p className="muted">
Select up to 16 logs. Each file is streamed separately and
retains at most 3,000 parsed records; the merged timeline
recognizes W3C trace/span IDs and common request or
correlation fields. First-event offsets are observations,
not clock-skew estimates.
</p>
</div>
<div className="actions">
<label className="button file-button">
Select log set
<input
type="file"
multiple
disabled={traceScanning}
onChange={(event) =>
void scanTraceFiles(event.target.files)
}
/>
</label>
{traceScanning && (
<button
type="button"
onClick={() => traceController.current?.abort()}
>
Cancel
</button>
)}
<button
type="button"
disabled={traceSources.length === 0}
onClick={() => {
try {
const value = exportOtlpJson(traceSources);
triggerBlobDownload(
new Blob([value], {
type: "application/json;charset=utf-8",
}),
"retained-logs.otlp.json",
);
setError("");
} catch (reason) {
setError(message(reason, "OTLP export failed."));
}
}}
>
Export retained OTLP/JSON
</button>
</div>
</div>
{traceProgress && (
<p className="notice" aria-live="polite">
{traceProgress}
</p>
)}
<div className="metrics trace-metrics">
<article>
<span>Files</span>
<strong>{traceAnalysis.files}</strong>
<small>
{multiReports.length ? "selected set" : "current log"}
</small>
</article>
<article>
<span>Records</span>
<strong>
{traceAnalysis.inputRecords.toLocaleString()}
</strong>
<small>
{traceAnalysis.droppedEvents.toLocaleString()} hidden by
merged-table cap
</small>
</article>
<article>
<span>Correlation groups</span>
<strong>
{traceAnalysis.groups.length.toLocaleString()}
</strong>
<small>trace ID preferred over request ID</small>
</article>
<article>
<span>Undated</span>
<strong>
{traceAnalysis.undatedEvents.toLocaleString()}
</strong>
<small>sorted after dated events</small>
</article>
</div>
{traceAnalysis.diagnostics.length > 0 && (
<ul className="diagnostics">
{traceAnalysis.diagnostics.map((diagnostic) => (
<li key={diagnostic}>{diagnostic}</li>
))}
</ul>
)}
<div className="correlation-grid multi-correlation-grid">
<section className="stack">
<h3>Source timing</h3>
<div className="table-wrap">
<table>
<thead>
<tr>
<th>File</th>
<th>First last</th>
<th>First-event offset</th>
<th>Backward jumps</th>
</tr>
</thead>
<tbody>
{traceAnalysis.sourceTiming.map((timing) => (
<tr key={timing.filename}>
<td>{timing.filename}</td>
<td>
{timing.firstTimestamp ?? "—"} {" "}
{timing.lastTimestamp ?? "—"}
</td>
<td>
{timing.firstObservedOffsetMs === undefined
? "—"
: `${timing.firstObservedOffsetMs >= 0 ? "+" : ""}${timing.firstObservedOffsetMs} ms`}
</td>
<td>{timing.backwardJumps}</td>
</tr>
))}
</tbody>
</table>
</div>
</section>
<section className="stack timeline-section">
<h3>Trace and correlation groups</h3>
<div className="table-wrap">
<table>
<thead>
<tr>
<th>Kind / ID</th>
<th>Events</th>
<th>Duration</th>
<th>Files / sources</th>
<th>Trace evidence</th>
</tr>
</thead>
<tbody>
{traceAnalysis.groups.slice(0, 500).map((group) => (
<tr key={group.key}>
<td>
<small>{group.kind}</small>
<br />
<code>{group.id}</code>
</td>
<td>
{group.events} · {group.errors} errors
</td>
<td>
{group.durationMs === undefined
? "—"
: `${group.durationMs.toLocaleString()} ms`}
</td>
<td>
{group.files.join(", ")} /{" "}
{group.sources.join(", ")}
</td>
<td>
{group.spanCount} spans
{group.unresolvedParentSpanIds.length
? ` · ${group.unresolvedParentSpanIds.length} parent IDs not present in retained logs`
: ""}
{group.repeatedSpanIds.length
? ` · ${group.repeatedSpanIds.length} span IDs link multiple events`
: ""}
</td>
</tr>
))}
</tbody>
</table>
</div>
{traceAnalysis.groups.length === 0 && (
<p className="empty">
No valid trace ID or common correlation/request ID was
found in the retained records.
</p>
)}
</section>
</div>
<section className="stack">
<h3>Merged event timeline</h3>
<div className="table-wrap trace-events">
<table>
<thead>
<tr>
<th>Time</th>
<th>File:line</th>
<th>Level / source</th>
<th>Correlation</th>
<th>Message</th>
</tr>
</thead>
<tbody>
{traceAnalysis.events.slice(0, 1_000).map((event) => (
<tr key={`${event.filename}-${event.line}`}>
<td>{event.timestamp ?? "undated"}</td>
<td>
{event.filename}:{event.line}
</td>
<td>
{event.level} · {event.source}
</td>
<td>
<code>
{event.traceId ?? event.correlationId ?? "—"}
</code>
</td>
<td>{event.message}</td>
</tr>
))}
</tbody>
</table>
</div>
{traceAnalysis.events.length > 1_000 && (
<p className="notice">
The page renders the first 1,000 merged events; analysis
and OTLP mapping use the bounded retained set.
</p>
)}
</section>
</div>
)}
{tab === "redact" && ( {tab === "redact" && (
<div className="redaction-grid" role="tabpanel"> <div className="redaction-grid">
<div className="stack"> <div className="stack">
<div> <div>
<p className="eyebrow">Deterministic recipes</p> <p className="eyebrow">Deterministic recipes</p>
@@ -696,16 +1106,17 @@ export function Workbench() {
)} )}
{tab === "export" && ( {tab === "export" && (
<div className="export-grid" role="tabpanel"> <div className="export-grid">
<div className="stack"> <div className="stack">
<div> <div>
<p className="eyebrow">Bounded output</p> <p className="eyebrow">Preview or complete second pass</p>
<h2>Export the retained filtered subset</h2> <h2>Export filtered local records</h2>
<p className="muted"> <p className="muted">
Exports use only the current bounded preview and Download the bounded preview immediately, or rescan the
filtersnever unseen records dropped after the preview complete retained source and stream every matching record.
cap. CSV cells are protected against spreadsheet formula Direct file writing is used when supported; otherwise the
execution. fallback Blob is capped at 256 MiB. CSV cells are
protected against spreadsheet formula execution.
</p> </p>
</div> </div>
<label className="field"> <label className="field">
@@ -735,13 +1146,30 @@ export function Workbench() {
</label> </label>
<div className="actions"> <div className="actions">
<button <button
className="primary"
type="button" type="button"
onClick={prepareExport} onClick={prepareExport}
disabled={!filtered.length} disabled={!filtered.length}
> >
Prepare and download Download preview
</button> </button>
<button
className="primary"
type="button"
onClick={() => void exportCompleteSource()}
disabled={!sourceBlob || exporting}
>
{exporting
? "Exporting complete source…"
: "Export complete source"}
</button>
{exporting && (
<button
type="button"
onClick={() => exportController.current?.abort()}
>
Cancel export
</button>
)}
</div> </div>
{lastExport && ( {lastExport && (
<p className={lastExport.truncated ? "warning" : "notice"}> <p className={lastExport.truncated ? "warning" : "notice"}>
@@ -754,6 +1182,29 @@ export function Workbench() {
. .
</p> </p>
)} )}
{exportProgress && exporting && (
<p className="notice" aria-live="polite">
{bytes(exportProgress.bytesRead)} of{" "}
{bytes(exportProgress.totalBytes)} read ·{" "}
{exportProgress.rows.toLocaleString()} matching rows ·{" "}
{bytes(exportProgress.bytesWritten)} written
</p>
)}
{lastFullExport && !exporting && (
<p
className={
lastFullExport.truncated ? "warning" : "notice"
}
>
Complete pass wrote {lastFullExport.rows.toLocaleString()}{" "}
row
{lastFullExport.rows === 1 ? "" : "s"} from{" "}
{lastFullExport.physicalLines.toLocaleString()} physical
lines · {bytes(lastFullExport.bytesWritten)}
{lastFullExport.truncated ? " · input limit reached" : ""}
.
</p>
)}
</div> </div>
<div className="summary-card"> <div className="summary-card">
<h3>Export boundary</h3> <h3>Export boundary</h3>
@@ -771,11 +1222,11 @@ export function Workbench() {
<dd>{filtered.length.toLocaleString()}</dd> <dd>{filtered.length.toLocaleString()}</dd>
</div> </div>
<div> <div>
<dt>Max rows</dt> <dt>Preview max rows</dt>
<dd>{LOG_LIMITS.exportRows.toLocaleString()}</dd> <dd>{LOG_LIMITS.exportRows.toLocaleString()}</dd>
</div> </div>
<div> <div>
<dt>Max text</dt> <dt>Preview max text</dt>
<dd>{bytes(LOG_LIMITS.exportChars)}</dd> <dd>{bytes(LOG_LIMITS.exportChars)}</dd>
</div> </div>
</dl> </dl>
+721 -15
View File
@@ -1,4 +1,8 @@
import { assertBoundedText, safeJsonParse } from "@add-ideas/toolbox-helpers"; import {
assertBoundedText,
safeJsonParse,
stableStringify,
} from "@add-ideas/toolbox-helpers";
export type LogFormat = export type LogFormat =
| "auto" | "auto"
@@ -10,6 +14,7 @@ export type LogFormat =
| "logfmt"; | "logfmt";
export interface LogRecord { export interface LogRecord {
line: number; line: number;
lineEnd?: number;
parser: Exclude<LogFormat, "auto">; parser: Exclude<LogFormat, "auto">;
valid: boolean; valid: boolean;
raw: string; raw: string;
@@ -63,7 +68,10 @@ export interface ScanOptions {
filename?: string; filename?: string;
previewLimit?: number; previewLimit?: number;
maxLines?: number; maxLines?: number;
multiline?: MultilineMode;
} }
export type MultilineMode =
"off" | "auto" | "java" | "dotnet" | "python" | "indented";
export interface LogFilters { export interface LogFilters {
query?: string; query?: string;
level?: string; level?: string;
@@ -94,6 +102,36 @@ export interface ExportResult {
extension: "csv" | "ndjson" | "log"; extension: "csv" | "ndjson" | "log";
} }
export interface FullExportOptions {
format: "csv" | "ndjson" | "text";
inputFormat: Exclude<LogFormat, "auto">;
filters?: LogFilters;
redaction?: RedactionOptions;
multiline?: MultilineMode;
maxLines?: number;
maxBlobBytes?: number;
writable?: WritableStream<Uint8Array>;
}
export interface FullExportProgress {
bytesRead: number;
totalBytes: number;
physicalLines: number;
rows: number;
bytesWritten: number;
}
export interface FullExportResult {
blob?: Blob;
rows: number;
physicalLines: number;
bytesRead: number;
bytesWritten: number;
truncated: boolean;
mediaType: string;
extension: "csv" | "ndjson" | "log";
}
export const LOG_LIMITS = { export const LOG_LIMITS = {
fileBytes: 8 * 1024 * 1024 * 1024, fileBytes: 8 * 1024 * 1024 * 1024,
pasteChars: 2 * 1024 * 1024, pasteChars: 2 * 1024 * 1024,
@@ -109,6 +147,7 @@ export const LOG_LIMITS = {
timelineBuckets: 2_000, timelineBuckets: 2_000,
exportRows: 50_000, exportRows: 50_000,
exportChars: 10 * 1024 * 1024, exportChars: 10 * 1024 * 1024,
fullExportBlobBytes: 256 * 1024 * 1024,
} as const; } as const;
const ANSI = new RegExp(String.raw`\u001B(?:\[[0-?]*[ -/]*[@-~]|[@-_])`, "gu"); const ANSI = new RegExp(String.raw`\u001B(?:\[[0-?]*[ -/]*[@-~]|[@-_])`, "gu");
@@ -689,6 +728,65 @@ class LineFramer {
} }
} }
interface LogicalLine {
value: string;
oversized: boolean;
line: number;
lineEnd: number;
}
function continuationLine(value: string, mode: MultilineMode): boolean {
if (mode === "off") return false;
if (mode === "indented") return /^[ \t]/u.test(value);
const java = /^\s+(?:at\s|\.\.\. \d+ more)|^(?:Caused by|Suppressed):/u.test(
value,
);
const dotnet = /^\s+at\s|^--- End of .* stack trace ---/u.test(value);
const python =
/^(?:Traceback \(most recent call last\):|\s+File ".*", line \d+|\s{4,}\S|\s+\^|During handling of the above exception|The above exception was the direct cause)/u.test(
value,
);
if (mode === "java") return java;
if (mode === "dotnet") return dotnet;
if (mode === "python") return python;
return java || dotnet || python;
}
class MultilineAssembler {
private current?: LogicalLine;
private readonly mode: MultilineMode;
constructor(mode: MultilineMode) {
this.mode = mode;
}
add(
value: string,
oversized: boolean,
line: number,
): LogicalLine | undefined {
if (this.current && continuationLine(value, this.mode)) {
const separator = this.current.value.length ? "\n" : "";
const remaining = LOG_LIMITS.lineChars - this.current.value.length;
const addition = `${separator}${value}`;
if (addition.length > remaining) this.current.oversized = true;
if (remaining > 0) this.current.value += addition.slice(0, remaining);
this.current.oversized ||= oversized;
this.current.lineEnd = line;
return undefined;
}
const completed = this.current;
this.current = { value, oversized, line, lineEnd: line };
return completed;
}
finish(): LogicalLine | undefined {
const completed = this.current;
this.current = undefined;
return completed;
}
}
function increment(map: Map<string, number>, key: string) { function increment(map: Map<string, number>, key: string) {
const safe = key.trim().slice(0, 160) || "unknown"; const safe = key.trim().slice(0, 160) || "unknown";
if (map.has(safe)) map.set(safe, map.get(safe)! + 1); if (map.has(safe)) map.set(safe, map.get(safe)! + 1);
@@ -713,6 +811,7 @@ class Accumulator {
private confidence = 1; private confidence = 1;
private readonly detection: Array<{ private readonly detection: Array<{
line: number; line: number;
lineEnd: number;
value: string; value: string;
oversized: boolean; oversized: boolean;
}> = []; }> = [];
@@ -739,20 +838,21 @@ class Accumulator {
this.requested = requested; this.requested = requested;
if (requested !== "auto") this.format = requested; if (requested !== "auto") this.format = requested;
} }
add(value: string, oversized: boolean) { add(item: LogicalLine) {
this.totalLines += 1; this.totalLines += item.lineEnd - item.line + 1;
if (this.totalLines === 1 && value.charCodeAt(0) === 0xfeff) let { value } = item;
if (item.line === 1 && value.charCodeAt(0) === 0xfeff)
value = value.slice(1); value = value.slice(1);
if (!value.trim() && !oversized) { if (!value.trim() && !item.oversized) {
this.blankLines += 1; this.blankLines += item.lineEnd - item.line + 1;
return; return;
} }
const current = { line: this.totalLines, value, oversized }; const current = { ...item, value };
if (!this.format) { if (!this.format) {
this.detection.push(current); this.detection.push(current);
if (this.detection.length < 30) return; if (this.detection.length < 30) return;
const detected = detectLogFormat( const detected = detectLogFormat(
this.detection.map((item) => item.value), this.detection.map((entry) => entry.value.split("\n", 1)[0] ?? ""),
); );
this.format = detected.format; this.format = detected.format;
this.confidence = detected.confidence; this.confidence = detected.confidence;
@@ -764,7 +864,7 @@ class Accumulator {
finish() { finish() {
if (!this.format) { if (!this.format) {
const detected = detectLogFormat( const detected = detectLogFormat(
this.detection.map((item) => item.value), this.detection.map((entry) => entry.value.split("\n", 1)[0] ?? ""),
); );
this.format = detected.format; this.format = detected.format;
this.confidence = detected.confidence; this.confidence = detected.confidence;
@@ -816,7 +916,7 @@ class Accumulator {
lastTimestamp: this.lastTimestamp, lastTimestamp: this.lastTimestamp,
}; };
} }
private consume(item: { line: number; value: string; oversized: boolean }) { private consume(item: LogicalLine) {
const record = item.oversized const record = item.oversized
? issueRecord( ? issueRecord(
item.line, item.line,
@@ -824,7 +924,8 @@ class Accumulator {
this.format!, this.format!,
`Line exceeded ${LOG_LIMITS.lineChars.toLocaleString()} characters and was truncated.`, `Line exceeded ${LOG_LIMITS.lineChars.toLocaleString()} characters and was truncated.`,
) )
: parseLogLine(item.value, item.line, this.format!); : parseLogicalLogLine(item.value, item.line, this.format!);
if (item.lineEnd > item.line) record.lineEnd = item.lineEnd;
if (item.oversized) this.oversizedLines += 1; if (item.oversized) this.oversizedLines += 1;
if (record.valid) this.parsedLines += 1; if (record.valid) this.parsedLines += 1;
else this.invalidLines += 1; else this.invalidLines += 1;
@@ -870,6 +971,25 @@ class Accumulator {
} }
} }
function parseLogicalLogLine(
value: string,
line: number,
format: Exclude<LogFormat, "auto">,
): LogRecord {
const newline = value.indexOf("\n");
if (newline < 0) return parseLogLine(value, line, format);
const head = value.slice(0, newline);
const continuation = value.slice(newline + 1);
const record = parseLogLine(head, line, format);
return {
...record,
raw: retainedText(value),
message: retainedText(
`${record.message}${record.message ? "\n" : ""}${continuation}`,
),
};
}
export async function scanLogBlob( export async function scanLogBlob(
blob: Blob, blob: Blob,
options: ScanOptions = {}, options: ScanOptions = {},
@@ -889,10 +1009,17 @@ export async function scanLogBlob(
const accumulator = new Accumulator(previewLimit, options.format ?? "auto"); const accumulator = new Accumulator(previewLimit, options.format ?? "auto");
const decoder = new TextDecoder("utf-8", { fatal: false }); const decoder = new TextDecoder("utf-8", { fatal: false });
const framer = new LineFramer(); const framer = new LineFramer();
const assembler = new MultilineAssembler(options.multiline ?? "auto");
const reader = blob.stream().getReader(); const reader = blob.stream().getReader();
let bytesRead = 0; let bytesRead = 0;
let progressAt = 0; let progressAt = 0;
let stoppedReason: string | undefined; let stoppedReason: string | undefined;
let physicalLine = 0;
const acceptPhysicalLine = (line: { value: string; oversized: boolean }) => {
physicalLine += 1;
const completed = assembler.add(line.value, line.oversized, physicalLine);
if (completed) accumulator.add(completed);
};
const checkAbort = () => { const checkAbort = () => {
if (signal?.aborted) throw new ScanCancelledError(); if (signal?.aborted) throw new ScanCancelledError();
}; };
@@ -905,8 +1032,8 @@ export async function scanLogBlob(
for (const line of framer.feed( for (const line of framer.feed(
decoder.decode(chunk.value, { stream: true }), decoder.decode(chunk.value, { stream: true }),
)) { )) {
accumulator.add(line.value, line.oversized); acceptPhysicalLine(line);
if (accumulator.totalLines >= maxLines) { if (physicalLine >= maxLines) {
stoppedReason = `Stopped after the ${maxLines.toLocaleString()}-line safety limit.`; stoppedReason = `Stopped after the ${maxLines.toLocaleString()}-line safety limit.`;
await reader.cancel(stoppedReason); await reader.cancel(stoppedReason);
break; break;
@@ -927,9 +1054,10 @@ export async function scanLogBlob(
} }
if (!stoppedReason) { if (!stoppedReason) {
const tail = decoder.decode(); const tail = decoder.decode();
for (const line of framer.feed(tail, true)) for (const line of framer.feed(tail, true)) acceptPhysicalLine(line);
accumulator.add(line.value, line.oversized);
} }
const completed = assembler.finish();
if (completed) accumulator.add(completed);
checkAbort(); checkAbort();
onProgress?.({ onProgress?.({
bytesRead, bytesRead,
@@ -1202,3 +1330,581 @@ export function exportRecords(
extension: format === "text" ? "log" : format, extension: format === "text" ? "log" : format,
}; };
} }
function exportMediaType(format: "csv" | "ndjson" | "text"): string {
return format === "csv"
? "text/csv;charset=utf-8"
: format === "ndjson"
? "application/x-ndjson;charset=utf-8"
: "text/plain;charset=utf-8";
}
function fullExportRow(
record: LogRecord,
format: "csv" | "ndjson" | "text",
): string {
if (format === "ndjson") return `${JSON.stringify(record)}\n`;
if (format === "text") return `${record.raw}\n`;
return `${[
String(record.line),
String(record.lineEnd ?? record.line),
record.timestamp ?? "",
record.level,
record.source,
record.message,
record.parser,
String(record.valid),
JSON.stringify(record.fields),
]
.map(csvCell)
.join(",")}\r\n`;
}
/**
* Performs a second streaming pass over a log and writes every matching row.
* Passing a writable avoids retaining output in memory; otherwise a bounded
* Blob is returned for browsers without the File System Access API.
*/
export async function exportLogBlob(
blob: Blob,
options: FullExportOptions,
onProgress?: (progress: FullExportProgress) => void,
signal?: AbortSignal,
): Promise<FullExportResult> {
if (blob.size > LOG_LIMITS.fileBytes)
throw new RangeError("Log file exceeds the 8 GiB streaming limit.");
const maxLines = Math.max(
1,
Math.min(options.maxLines ?? LOG_LIMITS.maxLines, LOG_LIMITS.maxLines),
);
const maxBlobBytes = Math.max(
1024 * 1024,
Math.min(
options.maxBlobBytes ?? LOG_LIMITS.fullExportBlobBytes,
LOG_LIMITS.fullExportBlobBytes,
),
);
const decoder = new TextDecoder("utf-8", { fatal: false });
const encoder = new TextEncoder();
const framer = new LineFramer();
const assembler = new MultilineAssembler(options.multiline ?? "auto");
const reader = blob.stream().getReader();
const writer = options.writable?.getWriter();
const chunks: Uint8Array<ArrayBuffer>[] = [];
let buffer =
options.format === "csv"
? "line,lineEnd,timestamp,level,source,message,parser,valid,fields\r\n"
: "";
let physicalLines = 0;
let rows = 0;
let bytesRead = 0;
let bytesWritten = 0;
let progressAt = 0;
let truncated = false;
const checkAbort = () => {
if (signal?.aborted) throw new ScanCancelledError();
};
const flush = async () => {
if (!buffer) return;
const bytes = encoder.encode(buffer);
buffer = "";
if (!writer && bytesWritten + bytes.byteLength > maxBlobBytes) {
throw new RangeError(
`Full export exceeds the ${maxBlobBytes.toLocaleString()}-byte in-memory limit; use Save to file in a supporting browser.`,
);
}
if (writer) await writer.write(bytes);
else chunks.push(bytes);
bytesWritten += bytes.byteLength;
};
const consume = async (line: LogicalLine) => {
const record = parseLogicalLogLine(
line.value,
line.line,
options.inputFormat,
);
if (line.lineEnd > line.line) record.lineEnd = line.lineEnd;
if (!filterRecords([record], options.filters ?? {}).length) return;
const output = options.redaction
? redactRecords([record], options.redaction).records[0]!
: record;
buffer += fullExportRow(output, options.format);
rows += 1;
if (buffer.length >= 256 * 1024) await flush();
};
const accept = async (line: { value: string; oversized: boolean }) => {
physicalLines += 1;
const completed = assembler.add(line.value, line.oversized, physicalLines);
if (completed) await consume(completed);
};
try {
while (physicalLines < maxLines) {
checkAbort();
const chunk = await reader.read();
if (chunk.done) break;
bytesRead += chunk.value.byteLength;
for (const line of framer.feed(
decoder.decode(chunk.value, { stream: true }),
)) {
await accept(line);
if (physicalLines >= maxLines) {
truncated = true;
await reader.cancel("Full export reached its physical-line limit.");
break;
}
}
if (bytesRead - progressAt >= 1024 * 1024 || bytesRead === blob.size) {
await flush();
progressAt = bytesRead;
onProgress?.({
bytesRead,
totalBytes: blob.size,
physicalLines,
rows,
bytesWritten,
});
await new Promise<void>((resolve) => setTimeout(resolve, 0));
}
if (truncated) break;
}
if (!truncated) {
const tail = decoder.decode();
for (const line of framer.feed(tail, true)) await accept(line);
}
const completed = assembler.finish();
if (completed) await consume(completed);
await flush();
checkAbort();
if (writer) await writer.close();
const mediaType = exportMediaType(options.format);
onProgress?.({
bytesRead,
totalBytes: blob.size,
physicalLines,
rows,
bytesWritten,
});
return {
...(writer ? {} : { blob: new Blob(chunks, { type: mediaType }) }),
rows,
physicalLines,
bytesRead,
bytesWritten,
truncated,
mediaType,
extension: options.format === "text" ? "log" : options.format,
};
} catch (reason) {
await reader.cancel().catch(() => undefined);
if (writer) await writer.abort(reason).catch(() => undefined);
if (signal?.aborted || reason instanceof ScanCancelledError)
throw new ScanCancelledError();
throw reason;
} finally {
reader.releaseLock();
}
}
export interface NamedLogSource {
filename: string;
records: readonly LogRecord[];
}
export interface CorrelatedLogEvent {
filename: string;
line: number;
timestamp?: string;
epochMs?: number;
level: string;
source: string;
message: string;
traceId?: string;
spanId?: string;
parentSpanId?: string;
correlationId?: string;
}
export interface CorrelationGroup {
key: string;
kind: "trace" | "correlation";
id: string;
events: number;
errors: number;
sources: string[];
files: string[];
firstTimestamp?: string;
lastTimestamp?: string;
durationMs?: number;
spanCount: number;
unresolvedParentSpanIds: string[];
repeatedSpanIds: string[];
}
export interface MultiLogAnalysis {
files: number;
inputRecords: number;
retainedEvents: number;
undatedEvents: number;
droppedEvents: number;
events: CorrelatedLogEvent[];
groups: CorrelationGroup[];
sourceTiming: Array<{
filename: string;
firstTimestamp?: string;
lastTimestamp?: string;
firstObservedOffsetMs?: number;
backwardJumps: number;
}>;
diagnostics: string[];
}
const MULTI_FILE_LIMIT = 32;
const MULTI_RECORD_LIMIT = 50_000;
const MULTI_EVENT_RETAIN = 20_000;
/**
* Correlate bounded retained previews from several files. Offsets describe
* first observed events only and deliberately do not claim to estimate clock skew.
*/
export function analyzeLogSources(
sources: readonly NamedLogSource[],
): MultiLogAnalysis {
if (sources.length === 0)
return {
files: 0,
inputRecords: 0,
retainedEvents: 0,
undatedEvents: 0,
droppedEvents: 0,
events: [],
groups: [],
sourceTiming: [],
diagnostics: [],
};
if (sources.length > MULTI_FILE_LIMIT)
throw new Error(
`Multi-file analysis is limited to ${MULTI_FILE_LIMIT} files.`,
);
const names = new Set<string>();
const events: CorrelatedLogEvent[] = [];
const sourceTiming: MultiLogAnalysis["sourceTiming"] = [];
let inputRecords = 0;
let undatedEvents = 0;
let retainedTextCharacters = 0;
for (const input of sources) {
const filename = cleanFilename(input.filename);
if (names.has(filename))
throw new Error(`Duplicate source filename: ${filename}.`);
names.add(filename);
inputRecords += input.records.length;
if (inputRecords > MULTI_RECORD_LIMIT)
throw new Error(
`Multi-file analysis is limited to ${MULTI_RECORD_LIMIT.toLocaleString()} retained records.`,
);
let first: LogRecord | undefined;
let last: LogRecord | undefined;
let priorEpoch: number | undefined;
let backwardJumps = 0;
for (const record of input.records) {
retainedTextCharacters +=
record.raw.length +
record.message.length +
record.source.length +
Object.entries(record.fields).reduce(
(sum, [key, value]) => sum + key.length + value.length,
0,
);
if (retainedTextCharacters > 128 * 1024 * 1024)
throw new Error(
"Multi-file analysis exceeds the 128 MiB retained-text limit.",
);
if (record.epochMs === undefined) undatedEvents += 1;
else {
if (!first || record.epochMs < first.epochMs!) first = record;
if (!last || record.epochMs > last.epochMs!) last = record;
if (priorEpoch !== undefined && record.epochMs < priorEpoch)
backwardJumps += 1;
priorEpoch = record.epochMs;
}
events.push({
filename,
line: record.line,
...(record.timestamp ? { timestamp: record.timestamp } : {}),
...(record.epochMs === undefined ? {} : { epochMs: record.epochMs }),
level: record.level,
source: record.source,
message: record.message,
...correlationIdentifiers(record),
});
}
sourceTiming.push({
filename,
...(first?.timestamp ? { firstTimestamp: first.timestamp } : {}),
...(last?.timestamp ? { lastTimestamp: last.timestamp } : {}),
backwardJumps,
});
}
events.sort(
(left, right) =>
(left.epochMs ?? Number.POSITIVE_INFINITY) -
(right.epochMs ?? Number.POSITIVE_INFINITY) ||
left.filename.localeCompare(right.filename) ||
left.line - right.line,
);
const earliest = events.find((event) => event.epochMs !== undefined)?.epochMs;
if (earliest !== undefined)
for (const timing of sourceTiming) {
const first = timing.firstTimestamp
? Date.parse(timing.firstTimestamp)
: Number.NaN;
if (Number.isFinite(first))
timing.firstObservedOffsetMs = first - earliest;
}
const grouped = new Map<string, CorrelatedLogEvent[]>();
for (const event of events) {
const key = event.traceId
? `trace:${event.traceId}`
: event.correlationId
? `correlation:${event.correlationId}`
: "";
if (key) {
const group = grouped.get(key) ?? [];
group.push(event);
grouped.set(key, group);
}
}
const groups = [...grouped.entries()]
.map(([key, items]): CorrelationGroup => {
const kind = key.startsWith("trace:") ? "trace" : "correlation";
const dated = items.filter(
(item): item is CorrelatedLogEvent & { epochMs: number } =>
item.epochMs !== undefined,
);
const spanCounts = new Map<string, number>();
for (const item of items)
if (item.spanId)
spanCounts.set(item.spanId, (spanCounts.get(item.spanId) ?? 0) + 1);
const spanIds = new Set(spanCounts.keys());
const unresolvedParentSpanIds = [
...new Set(
items.flatMap((item) =>
item.parentSpanId && !spanIds.has(item.parentSpanId)
? [item.parentSpanId]
: [],
),
),
].sort();
const first = dated.at(0);
const last = dated.at(-1);
return {
key,
kind,
id: key.slice(key.indexOf(":") + 1),
events: items.length,
errors: items.filter((item) =>
["emergency", "alert", "critical", "error"].includes(item.level),
).length,
sources: [...new Set(items.map((item) => item.source))].sort(),
files: [...new Set(items.map((item) => item.filename))].sort(),
...(first?.timestamp ? { firstTimestamp: first.timestamp } : {}),
...(last?.timestamp ? { lastTimestamp: last.timestamp } : {}),
...(first && last ? { durationMs: last.epochMs - first.epochMs } : {}),
spanCount: spanIds.size,
unresolvedParentSpanIds,
repeatedSpanIds: [...spanCounts]
.filter(([, count]) => count > 1)
.map(([span]) => span)
.sort(),
};
})
.sort(
(left, right) =>
right.events - left.events || left.key.localeCompare(right.key),
)
.slice(0, 5_000);
const retainedEvents = Math.min(events.length, MULTI_EVENT_RETAIN);
const diagnostics = [
...(undatedEvents
? [
`${undatedEvents.toLocaleString()} retained event(s) have no parseable timestamp and sort after dated events.`,
]
: []),
...sourceTiming.flatMap((timing) =>
timing.backwardJumps
? [
`${timing.filename}: ${timing.backwardJumps.toLocaleString()} backward timestamp jump(s) in original record order.`,
]
: [],
),
...(events.length > retainedEvents
? [
`The merged event table retains ${retainedEvents.toLocaleString()} of ${events.length.toLocaleString()} events. Group statistics still use all bounded input records.`,
]
: []),
];
return {
files: sources.length,
inputRecords,
retainedEvents,
undatedEvents,
droppedEvents: events.length - retainedEvents,
events: events.slice(0, retainedEvents),
groups,
sourceTiming: sourceTiming.sort((left, right) =>
left.filename.localeCompare(right.filename),
),
diagnostics,
};
}
function correlationIdentifiers(
record: LogRecord,
): Pick<
CorrelatedLogEvent,
"traceId" | "spanId" | "parentSpanId" | "correlationId"
> {
const values = new Map<string, string>();
for (const [key, value] of Object.entries(record.fields))
values.set(key.toLowerCase().replaceAll(/[^a-z0-9]/gu, ""), value.trim());
const lookup = (names: readonly string[]) => {
for (const name of names) {
const value = values.get(name);
if (value) return value;
}
return undefined;
};
const traceCandidate =
lookup(["traceid", "oteltraceid"]) ??
/\btrace[_-]?id[=:"\s]+([0-9a-f]{32})\b/iu.exec(record.message)?.[1];
const spanCandidate =
lookup(["spanid", "otelspanid"]) ??
/\bspan[_-]?id[=:"\s]+([0-9a-f]{16})\b/iu.exec(record.message)?.[1];
const parentCandidate = lookup([
"parentspanid",
"parentid",
"otelparentspanid",
]);
const correlationCandidate = lookup([
"correlationid",
"requestid",
"requesttraceid",
"transactionid",
"operationid",
]);
const hex = (value: string | undefined, length: number) =>
value &&
new RegExp(`^[0-9a-f]{${length}}$`, "iu").test(value) &&
!/^0+$/u.test(value)
? value.toLowerCase()
: undefined;
const correlationId =
correlationCandidate &&
correlationCandidate.length >= 4 &&
correlationCandidate.length <= 128 &&
/^[A-Za-z0-9._:@/-]+$/u.test(correlationCandidate)
? correlationCandidate
: undefined;
return {
...(hex(traceCandidate, 32) ? { traceId: hex(traceCandidate, 32) } : {}),
...(hex(spanCandidate, 16) ? { spanId: hex(spanCandidate, 16) } : {}),
...(hex(parentCandidate, 16)
? { parentSpanId: hex(parentCandidate, 16) }
: {}),
...(correlationId ? { correlationId } : {}),
};
}
function cleanFilename(value: string): string {
const filename = cleanText(value).trim().slice(0, 512);
if (!filename) throw new Error("Every log source needs a filename.");
return filename;
}
/** Export retained records as a focused OTLP/JSON ExportLogsServiceRequest. */
export function exportOtlpJson(sources: readonly NamedLogSource[]): string {
const analysis = analyzeLogSources(sources);
if (analysis.inputRecords > 20_000)
throw new Error(
"OTLP/JSON export is limited to 20,000 retained records; reduce the selected set.",
);
let exportCharacters = 0;
const resourceLogs = sources.map((input) => {
const filename = cleanFilename(input.filename);
return {
resource: {
attributes: [
{
key: "service.name",
value: { stringValue: filename },
},
{
key: "toolbox.log.source",
value: { stringValue: "retained-preview" },
},
],
},
scopeLogs: [
{
scope: { name: "de.add-ideas.log-tools", version: "0.2.0" },
logRecords: input.records.map((record) => {
exportCharacters +=
record.message.length +
Object.entries(record.fields).reduce(
(sum, [key, value]) => sum + key.length + value.length,
0,
);
if (exportCharacters > 64 * 1024 * 1024)
throw new Error(
"OTLP/JSON output exceeds the 64 MiB text limit.",
);
const correlation = correlationIdentifiers(record);
return {
...(record.epochMs === undefined
? {}
: {
timeUnixNano: String(Math.trunc(record.epochMs)) + "000000",
}),
severityNumber: otlpSeverityNumber(record.level),
severityText: record.level.toUpperCase(),
body: { stringValue: record.message },
attributes: Object.entries(record.fields)
.slice(0, 100)
.map(([key, value]) => ({
key,
value: { stringValue: value },
})),
...(correlation.traceId ? { traceId: correlation.traceId } : {}),
...(correlation.spanId ? { spanId: correlation.spanId } : {}),
...(correlation.traceId ? { flags: 1 } : {}),
};
}),
},
],
};
});
return (
stableStringify({ resourceLogs }, 2, {
maxDepth: 16,
maxNodes: 10_000_000,
maxTextChars: 256 * 1024 * 1024,
}) + "\n"
);
}
function otlpSeverityNumber(level: string): number {
const values: Record<string, number> = {
trace: 1,
debug: 5,
info: 9,
notice: 10,
warning: 13,
error: 17,
critical: 21,
alert: 22,
emergency: 24,
};
return values[level] ?? 0;
}
+26 -1
View File
@@ -154,6 +154,12 @@ code {
align-items: end; align-items: end;
margin-bottom: 0.9rem; margin-bottom: 0.9rem;
} }
.compact-heading {
align-items: flex-start;
}
.compact-heading > div:first-child {
max-width: 58rem;
}
.source-grid { .source-grid {
display: grid; display: grid;
grid-template-columns: minmax(15rem, 0.75fr) minmax(20rem, 1.25fr); grid-template-columns: minmax(15rem, 0.75fr) minmax(20rem, 1.25fr);
@@ -243,7 +249,7 @@ meter {
overflow-x: auto; overflow-x: auto;
padding-bottom: 0.2rem; padding-bottom: 0.2rem;
} }
.workspace-tabs button[aria-selected="true"] { .workspace-tabs button[aria-pressed="true"] {
border-color: var(--toolbox-accent); border-color: var(--toolbox-accent);
background: var(--toolbox-accent); background: var(--toolbox-accent);
color: var(--toolbox-accent-contrast); color: var(--toolbox-accent-contrast);
@@ -340,6 +346,19 @@ td small {
grid-template-columns: repeat(2, minmax(0, 1fr)); grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 1rem; gap: 1rem;
} }
.multi-correlation-grid {
grid-template-columns: 1fr;
}
.trace-events {
max-height: 36rem;
}
.trace-events td:last-child {
min-width: 20rem;
white-space: pre-wrap;
}
.trace-metrics {
grid-template-columns: repeat(4, minmax(0, 1fr));
}
.timeline-section { .timeline-section {
grid-column: 1 / -1; grid-column: 1 / -1;
} }
@@ -519,6 +538,9 @@ td small {
.progress-row { .progress-row {
grid-template-columns: 1fr; grid-template-columns: 1fr;
} }
.trace-metrics {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
} }
@media (max-width: 44rem) { @media (max-width: 44rem) {
.hero { .hero {
@@ -534,4 +556,7 @@ td small {
.panel { .panel {
padding: 0.75rem; padding: 0.75rem;
} }
.trace-metrics {
grid-template-columns: 1fr;
}
} }
+39 -1
View File
@@ -3,7 +3,7 @@
"schemaVersion": 1, "schemaVersion": 1,
"id": "de.add-ideas.log-tools", "id": "de.add-ideas.log-tools",
"name": "Log Tools", "name": "Log Tools",
"version": "0.1.0", "version": "0.2.0",
"description": "Stream, inspect, correlate, redact, and export logs locally.", "description": "Stream, inspect, correlate, redact, and export logs locally.",
"entry": "./", "entry": "./",
"icon": "./favicon.svg", "icon": "./favicon.svg",
@@ -21,6 +21,44 @@
"crossOriginIsolated": false, "crossOriginIsolated": false,
"topLevelContext": false "topLevelContext": false
}, },
"io": {
"accepts": [
{
"mediaType": "text/plain",
"extensions": [".log", ".txt"]
},
{
"mediaType": "application/x-ndjson",
"extensions": [".ndjson", ".jsonl"]
},
{
"mediaType": "application/json",
"extensions": [".json"]
}
],
"produces": [
{
"mediaType": "text/plain",
"extensions": [".log", ".txt"]
},
{
"mediaType": "application/x-ndjson",
"extensions": [".ndjson"]
},
{
"mediaType": "text/csv",
"extensions": [".csv"]
},
{
"mediaType": "application/json",
"extensions": [".json"]
}
]
},
"capabilities": {
"required": [],
"optional": ["file-system-access"]
},
"privacy": { "privacy": {
"processing": "local", "processing": "local",
"fileUploads": true, "fileUploads": true,
+1 -1
View File
@@ -1 +1 @@
export const APP_VERSION = "0.1.0"; export const APP_VERSION = "0.2.0";
+3 -3
View File
@@ -40,11 +40,11 @@ test("streams, filters, correlates, and redacts the sample locally", async ({
).toBeVisible(); ).toBeVisible();
await page.getByLabel("Literal search").fill("timeout"); await page.getByLabel("Literal search").fill("timeout");
await expect(page.getByText(/1 of 4 retained records match/u)).toBeVisible(); await expect(page.getByText(/1 of 4 retained records match/u)).toBeVisible();
await page.getByRole("tab", { name: "Correlation" }).click(); await page.getByRole("button", { name: "Correlation" }).click();
await expect( await expect(
page.getByRole("heading", { name: "Hourly timeline" }), page.getByRole("heading", { name: "Hourly timeline" }),
).toBeVisible(); ).toBeVisible();
await page.getByRole("tab", { name: "Redaction" }).click(); await page.getByRole("button", { name: "Redaction" }).click();
await expect(page.getByLabel("Redacted preview")).not.toContainText( await expect(page.getByLabel("Redacted preview")).not.toContainText(
"ada@example.test", "ada@example.test",
); );
@@ -77,7 +77,7 @@ test("serves release identity and hardened headers", async ({ request }) => {
const manifest = await request.get("/deep/nested/log/toolbox-app.json"); const manifest = await request.get("/deep/nested/log/toolbox-app.json");
await expect(manifest.json()).resolves.toMatchObject({ await expect(manifest.json()).resolves.toMatchObject({
id: "de.add-ideas.log-tools", id: "de.add-ideas.log-tools",
version: "0.1.0", version: "0.2.0",
entry: "./", entry: "./",
privacy: { processing: "local", telemetry: false }, privacy: { processing: "local", telemetry: false },
}); });
+18
View File
@@ -0,0 +1,18 @@
import { expect, test } from "@playwright/test";
test("keeps the primary workspace inside a narrow viewport", async ({
page,
}) => {
await page.goto("/deep/nested/log/");
await expect(page.locator("main").first()).toBeVisible();
await expect(
page.locator("main .loading, main .workbench-loading"),
).toHaveCount(0);
const widths = await page.evaluate(() => ({
content: document.documentElement.scrollWidth,
viewport: document.documentElement.clientWidth,
}));
expect(widths.viewport).toBeLessThanOrEqual(430);
expect(widths.content).toBeLessThanOrEqual(widths.viewport + 1);
});
+6 -2
View File
@@ -1,6 +1,6 @@
import { render, screen } from "@testing-library/react"; import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event"; import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
import { App } from "../../src/App"; import { App } from "../../src/App";
class StreamingTestBlob { class StreamingTestBlob {
@@ -25,6 +25,10 @@ class StreamingTestBlob {
} }
} }
afterEach(() => {
vi.unstubAllGlobals();
});
describe("Log Tools", () => { describe("Log Tools", () => {
it("streams the sample and exposes correlation and redaction views", async () => { it("streams the sample and exposes correlation and redaction views", async () => {
vi.stubGlobal("Blob", StreamingTestBlob as unknown as typeof Blob); vi.stubGlobal("Blob", StreamingTestBlob as unknown as typeof Blob);
@@ -52,7 +56,7 @@ describe("Log Tools", () => {
expect( expect(
screen.getByText("JSON Lines", { selector: ".metrics strong" }), screen.getByText("JSON Lines", { selector: ".metrics strong" }),
).toBeVisible(); ).toBeVisible();
await userEvent.click(screen.getByRole("tab", { name: "Redaction" })); await userEvent.click(screen.getByRole("button", { name: "Redaction" }));
expect(screen.getByText("Deterministic recipes")).toBeVisible(); expect(screen.getByText("Deterministic recipes")).toBeVisible();
expect(screen.getByLabelText("Redacted preview")).not.toHaveTextContent( expect(screen.getByLabelText("Redacted preview")).not.toHaveTextContent(
"ada@example.test", "ada@example.test",
+168
View File
@@ -1,8 +1,11 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { import {
ScanCancelledError, ScanCancelledError,
analyzeLogSources,
detectLogFormat, detectLogFormat,
exportRecords, exportRecords,
exportLogBlob,
exportOtlpJson,
filterRecords, filterRecords,
parseLogLine, parseLogLine,
redactRecords, redactRecords,
@@ -34,6 +37,17 @@ function streamingBlob(source: string, chunkSize = 7): Blob {
} as unknown as Blob; } as unknown as Blob;
} }
async function readBlobText(blob: Blob): Promise<string> {
return await new Promise((resolve, reject) => {
const reader = new FileReader();
reader.addEventListener("load", () => resolve(String(reader.result ?? "")));
reader.addEventListener("error", () =>
reject(reader.error ?? new Error("Could not read test Blob.")),
);
reader.readAsText(blob);
});
}
describe("incremental scanning", () => { describe("incremental scanning", () => {
it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => { it("uses Blob.stream and incremental TextDecoder across UTF-8 chunk boundaries", async () => {
const source = [ const source = [
@@ -85,6 +99,29 @@ describe("incremental scanning", () => {
expect(report.timeline.length).toBeGreaterThan(0); expect(report.timeline.length).toBeGreaterThan(0);
}); });
it("assembles Java, .NET, and Python continuation lines with provenance", async () => {
const source = [
"2026-09-01T10:00:00Z ERROR request failed",
" at service.Handler.run(Handler.java:42)",
"Caused by: java.io.IOException: disk",
"2026-09-01T10:00:01Z INFO recovered",
"Traceback (most recent call last):",
' File "worker.py", line 4, in run',
" raise RuntimeError()",
"RuntimeError",
].join("\n");
const report = await scanLogBlob(streamingBlob(source), {
format: "plain",
multiline: "auto",
});
expect(report.totalLines).toBe(8);
expect(report.preview[0]).toMatchObject({ line: 1, lineEnd: 3 });
expect(report.preview[0]?.message).toContain("Caused by");
expect(report.preview[1]).toMatchObject({ line: 4, lineEnd: 7 });
expect(report.preview[1]?.message).toContain("worker.py");
expect(report.preview[2]).toMatchObject({ line: 8 });
});
it("supports cancellation without returning a partial replacement report", async () => { it("supports cancellation without returning a partial replacement report", async () => {
const controller = new AbortController(); const controller = new AbortController();
controller.abort(); controller.abort();
@@ -259,4 +296,135 @@ describe("filter, redact, and export", () => {
).toMatchObject({ line: 1, parser: "jsonl" }); ).toMatchObject({ line: 1, parser: "jsonl" });
expect(exportRecords(records, "text").value).toContain("ada@example.test"); expect(exportRecords(records, "text").value).toContain("ada@example.test");
}); });
it("performs a complete filtered and redacted second streaming pass", async () => {
const source = Array.from({ length: 250 }, (_unused, index) =>
JSON.stringify({
level: index % 2 ? "info" : "error",
service: "api",
message: `row ${index} from ada@example.test`,
}),
).join("\n");
const progress: number[] = [];
const result = await exportLogBlob(
streamingBlob(source, 31),
{
format: "ndjson",
inputFormat: "jsonl",
filters: { level: "error" },
redaction: options,
},
(value) => progress.push(value.bytesRead),
);
expect(result.rows).toBe(125);
expect(result.physicalLines).toBe(250);
expect(result.blob).toBeDefined();
const output = await readBlobText(result.blob!);
expect(output).not.toContain("ada@example.test");
expect(output.trim().split("\n")).toHaveLength(125);
expect(progress.at(-1)).toBe(source.length);
});
});
describe("multi-file temporal and trace analysis", () => {
const traceId = "0123456789abcdef0123456789abcdef";
const rootSpan = "0123456789abcdef";
const childSpan = "fedcba9876543210";
const apiRecords = [
parseLogLine(
JSON.stringify({
timestamp: "2026-09-01T10:00:00.000Z",
level: "info",
service: "api",
message: "start",
trace_id: traceId,
span_id: rootSpan,
}),
1,
"jsonl",
),
parseLogLine(
JSON.stringify({
timestamp: "2026-09-01T09:59:59.900Z",
level: "error",
service: "api",
message: "late arrival",
trace_id: traceId,
span_id: rootSpan,
}),
2,
"jsonl",
),
];
const workerRecords = [
parseLogLine(
JSON.stringify({
timestamp: "2026-09-01T10:00:00.250Z",
level: "info",
service: "worker",
message: "child",
traceId,
spanId: childSpan,
parentSpanId: rootSpan,
}),
1,
"jsonl",
),
];
it("merges time order and correlates trace/span evidence across files", () => {
const analysis = analyzeLogSources([
{ filename: "api.log", records: apiRecords },
{ filename: "worker.log", records: workerRecords },
]);
expect(analysis).toMatchObject({
files: 2,
inputRecords: 3,
undatedEvents: 0,
});
expect(analysis.events[0]).toMatchObject({
filename: "api.log",
line: 2,
traceId,
});
expect(analysis.groups[0]).toMatchObject({
kind: "trace",
id: traceId,
events: 3,
errors: 1,
spanCount: 2,
unresolvedParentSpanIds: [],
repeatedSpanIds: [rootSpan],
});
expect(
analysis.sourceTiming.find((item) => item.filename === "api.log")
?.backwardJumps,
).toBe(1);
});
it("maps retained records to bounded OTLP/JSON without inventing timestamps", () => {
const value = exportOtlpJson([
{ filename: "api.log", records: apiRecords },
{
filename: "undated.log",
records: [parseLogLine("INFO no clock", 1, "plain")],
},
]);
const parsed = JSON.parse(value) as {
resourceLogs: Array<{
scopeLogs: Array<{
logRecords: Array<Record<string, unknown>>;
}>;
}>;
};
const first = parsed.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!;
expect(first).toMatchObject({
traceId,
spanId: rootSpan,
severityNumber: 9,
timeUnixNano: "1788256800000000000",
});
const undated = parsed.resourceLogs[1]!.scopeLogs[0]!.logRecords[0]!;
expect(undated).not.toHaveProperty("timeUnixNano");
});
}); });