From f209bf78ee37d2fad9b6b7be2f6c2dab49f273fb Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Wed, 2 Sep 2026 08:33:59 +0200 Subject: [PATCH] Release Log Tools 0.2.0 --- .gitea/workflows/verify.yml | 39 ++ CHANGELOG.md | 8 + README.md | 9 +- SOURCE.md | 4 +- THIRD_PARTY_NOTICES.md | 8 +- docs/ARCHITECTURE.md | 6 +- package-lock.json | 45 +- package.json | 10 +- playwright.config.ts | 22 +- public/CHANGELOG.md | 8 + public/LICENSES/npm-runtime-licenses.txt | 6 +- public/README.md | 9 +- public/SOURCE.md | 4 +- public/THIRD_PARTY_NOTICES.md | 8 +- public/docs/ARCHITECTURE.md | 6 +- public/sw.js | 2 +- public/toolbox-app.json | 19 +- src/components/HelpDialog.tsx | 6 + src/components/Workbench.tsx | 503 +++++++++++++++- src/log/model.ts | 736 ++++++++++++++++++++++- src/styles.css | 27 +- src/toolbox/manifest.source.json | 40 +- src/version.ts | 2 +- tests/browser/app.spec.ts | 6 +- tests/browser/responsive.spec.ts | 18 + tests/components/app.test.tsx | 8 +- tests/log/model.test.ts | 168 ++++++ 27 files changed, 1621 insertions(+), 106 deletions(-) create mode 100644 .gitea/workflows/verify.yml create mode 100644 tests/browser/responsive.spec.ts diff --git a/.gitea/workflows/verify.yml b/.gitea/workflows/verify.yml new file mode 100644 index 0000000..b84a7cb --- /dev/null +++ b/.gitea/workflows/verify.yml @@ -0,0 +1,39 @@ +name: Verify + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: verify-${{ gitea.repository }}-${{ gitea.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CI: "true" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + - name: Select declared npm version + run: npm install --global npm@11.17.0 + - name: Install dependencies + run: npm ci + - name: Audit runtime dependencies + run: npm audit --omit=dev --audit-level=moderate + - name: Check, test, and build + run: npm run check + - name: Install browser engines + run: npx playwright install --with-deps chromium firefox webkit + - name: Browser tests + run: npm run test:browser diff --git a/CHANGELOG.md b/CHANGELOG.md index 4192d88..1131662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.2.0 - 2026-09-02 + +- Added configurable multiline assembly, multi-file temporal merge, trace/span + and request-ID correlation with bounded retained evidence. +- Added OTLP/JSON output and a complete second streaming pass for filtered, + redacted export, with direct file-system writing where available and a + bounded Blob fallback. + ## 0.1.0 - 2026-09-01 - Added genuinely incremental `File.stream()` / `TextDecoder` scanning with progress, cancellation, line framing, and bounded memory. diff --git a/README.md b/README.md index 2a97801..960eb9b 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,16 @@ Log Tools is a standalone local-first application in the [add·ideas Toolbox](ht - Incremental UTF-8 file scanning with byte/line progress, cancellation, a five-million-line stop, and an eight-GiB input gate - Bounded pasted/sample fallback and automatic or explicit plain, JSON Lines, nginx common/combined, syslog, and logfmt parsing +- Configurable streaming assembly for Java, .NET, Python, and generic indented multiline records, retaining their physical start/end lines - Normalized level, timestamp, source, message, and common structured fields with malformed-line diagnostics - Whole-scan bounded level/source/hour aggregation plus a 10,000-record retained preview for literal search and field filters +- Multi-file temporal merging for up to 16 separately streamed logs, with first-event offsets, backward-time diagnostics, W3C trace/span and common request-ID grouping, cross-file duration/source summaries, and retained-evidence caveats +- Focused OTLP/JSON log export with nanosecond timestamp strings, severity mapping, string attributes, trace/span context, and no invented timestamp for undated events - Deterministic email, IP, UUID, credential/query-secret, long-number, and exact-literal redaction recipes with generic or salted correlation tokens -- Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, with row/text caps and spreadsheet-safe CSV cells +- Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, plus a complete second streaming pass with cancellation and progress; supported browsers write directly to a selected file and others use a 256 MiB Blob fallback - Inert ANSI/HTML handling, responsive Toolbox shell themes, nested-path offline PWA support, and deterministic release archives -The preview retains up to the first 10,000 nonblank parsed or malformed lines and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Search, redaction preview, and export therefore cannot include later records dropped from the preview. Auto-detection is heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, JSONL flattening is capped, and multiline stack traces remain separate plain records. +The preview retains up to the first 10,000 nonblank parsed or malformed records and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Multi-file mode retains at most 3,000 records per file (50,000/128 MiB across the analysis, 20,000 merged table events), so missing parent spans mean “not present in retained logs,” not necessarily a broken trace. OTLP/JSON export is capped at 20,000 records/64 MiB of retained text. Interactive search and redaction preview operate on bounded sets, while complete single-source export reapplies the same filters and recipes during a second pass over every scanned physical line. Auto-detection and automatic multiline classification are heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, and JSONL flattening is capped. See [Architecture](docs/ARCHITECTURE.md) and [Privacy and security](docs/PRIVACY-SECURITY.md) for exact bounds and trust assumptions. @@ -30,7 +33,7 @@ npm run test:browser ## Release -`npm run release:artifact` creates deterministic `release/log-tools-0.1.0.zip` and checksum files. +`npm run release:artifact` creates deterministic `release/log-tools-0.2.0.zip` and checksum files. ## Licence diff --git a/SOURCE.md b/SOURCE.md index 343ab6c..311c15a 100644 --- a/SOURCE.md +++ b/SOURCE.md @@ -1,7 +1,7 @@ # Corresponding source -The corresponding source for Log Tools 0.1.0 is available at: +The corresponding source for Log Tools 0.2.0 is available at: -https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.1.0 +https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.2.0 Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index f52e239..8846877 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,12 +1,12 @@ # Third-party notices -Log Tools 0.1.0 directly depends on these runtime packages: +Log Tools 0.2.0 directly depends on these runtime packages: | Package | Pinned version | Declared licence | | -------------------------------- | -------------: | ---------------- | -| `@add-ideas/toolbox-contract` | 0.2.3 | Apache-2.0 | -| `@add-ideas/toolbox-helpers` | 0.1.0 | GPL-3.0-or-later | -| `@add-ideas/toolbox-shell-react` | 0.2.3 | Apache-2.0 | +| `@add-ideas/toolbox-contract` | 0.3.0 | Apache-2.0 | +| `@add-ideas/toolbox-helpers` | 0.2.0 | GPL-3.0-or-later | +| `@add-ideas/toolbox-shell-react` | 0.3.0 | Apache-2.0 | | `react` | 19.2.8 | MIT | | `react-dom` | 19.2.8 | MIT | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2c1f7be..f8aaae9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,9 +1,11 @@ # Architecture -Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string. +Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. A bounded multiline assembler groups configured stack-trace continuations and records their first and last physical lines. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string. Automatic detection buffers only the first 30 nonblank bounded lines, chooses among JSON Lines, nginx common/combined, RFC 5424/common RFC 3164 syslog, logfmt, and plain text, then replays that small buffer through the selected parser. Explicit selection skips detection. Parsing continues up to five million lines or eight GiB. At most the first 10,000 nonblank records and 32 MiB of retained characters are kept; individual retained raw/message strings, fields, level/source maps, hourly timestamp buckets, field names, counters, and progress are independently capped. Reads yield between progress intervals so cancellation and UI updates can run. -Filtering operates on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Exports use the filtered retained preview, at most 50,000 rows and 10 MiB of text. CSV prefixes spreadsheet-formula-leading cells. +Filtering operates interactively on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Quick exports use the preview; `exportLogBlob()` performs a second cancellable streaming pass and reapplies parsing, filters, and redaction to all records. It writes through a supplied `WritableStream`, or returns a Blob under a 256 MiB cap. CSV prefixes spreadsheet-formula-leading cells. + +Multi-file analysis streams each selected file through the same scanner, retains at most 3,000 records per file, and merges only those retained records. `analyzeLogSources()` caps file count, record count, retained characters, merged events, and correlation groups. It recognizes valid non-zero W3C-sized trace/span IDs plus bounded common request identifiers, reports unresolved parent references as retained-evidence gaps, and never adjusts timestamps. `exportOtlpJson()` produces a focused bounded OTLP/JSON request from retained records; it does not claim to reconstruct complete traces or export the unretained source. React renders log values through text nodes only. ANSI escape sequences are removed and other control bytes are replaced; HTML-like content is never interpreted. Relative assets and a same-origin scoped service worker keep the build relocatable below nested portal routes. There is no telemetry, database, server API, worker, or remote dependency at runtime. diff --git a/package-lock.json b/package-lock.json index 2bd6d5e..af587dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,22 +1,22 @@ { "name": "log-tools", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "log-tools", - "version": "0.1.0", + "version": "0.2.0", "license": "GPL-3.0-or-later", "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3", - "@add-ideas/toolbox-helpers": "0.1.0", - "@add-ideas/toolbox-shell-react": "0.2.3", + "@add-ideas/toolbox-contract": "0.3.0", + "@add-ideas/toolbox-helpers": "0.2.0", + "@add-ideas/toolbox-shell-react": "0.3.0", "react": "19.2.8", "react-dom": "19.2.8" }, "devDependencies": { - "@add-ideas/toolbox-testkit": "0.2.3", + "@add-ideas/toolbox-testkit": "0.3.0", "@eslint/js": "10.0.1", "@playwright/test": "1.62.1", "@testing-library/jest-dom": "6.9.1", @@ -42,24 +42,24 @@ } }, "node_modules/@add-ideas/toolbox-contract": { - "version": "0.2.3", - "license": "Apache-2.0", - "engines": { - "node": ">=20" - } + "version": "0.3.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz", + "integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==", + "license": "Apache-2.0" }, "node_modules/@add-ideas/toolbox-helpers": { - "version": "0.1.0", - "license": "GPL-3.0-or-later", - "engines": { - "node": ">=22" - } + "version": "0.2.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.2.0/toolbox-helpers-0.2.0.tgz", + "integrity": "sha512-SdOqkw+P+3J3fa5iVkzb5P15rVepB001GNV21Oh8w0CZcVL+YRltgD/s+MVcTyrNijWQf3E5vtQON/3N2LLyKg==", + "license": "GPL-3.0-or-later" }, "node_modules/@add-ideas/toolbox-shell-react": { - "version": "0.2.3", + "version": "0.3.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz", + "integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==", "license": "Apache-2.0", "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3" + "@add-ideas/toolbox-contract": "0.3.0" }, "peerDependencies": { "react": ">=18 <20", @@ -67,17 +67,16 @@ } }, "node_modules/@add-ideas/toolbox-testkit": { - "version": "0.2.3", + "version": "0.3.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz", + "integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3" + "@add-ideas/toolbox-contract": "0.3.0" }, "bin": { "toolbox-check": "dist/cli.js" - }, - "engines": { - "node": ">=20" } }, "node_modules/@adobe/css-tools": { diff --git a/package.json b/package.json index 88bc3f3..118366c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "log-tools", - "version": "0.1.0", + "version": "0.2.0", "description": "Stream, inspect, correlate, redact, and export logs locally in the browser.", "license": "GPL-3.0-or-later", "author": "Albrecht Degering", @@ -39,14 +39,14 @@ "release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force" }, "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3", - "@add-ideas/toolbox-helpers": "0.1.0", - "@add-ideas/toolbox-shell-react": "0.2.3", + "@add-ideas/toolbox-contract": "0.3.0", + "@add-ideas/toolbox-helpers": "0.2.0", + "@add-ideas/toolbox-shell-react": "0.3.0", "react": "19.2.8", "react-dom": "19.2.8" }, "devDependencies": { - "@add-ideas/toolbox-testkit": "0.2.3", + "@add-ideas/toolbox-testkit": "0.3.0", "@eslint/js": "10.0.1", "@playwright/test": "1.62.1", "@testing-library/jest-dom": "6.9.1", diff --git a/playwright.config.ts b/playwright.config.ts index d345d81..678757c 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -15,7 +15,25 @@ export default defineConfig({ timeout: 180_000, }, projects: [ - { name: "chromium", use: { ...devices["Desktop Chrome"] } }, - { name: "firefox", use: { ...devices["Desktop Firefox"] } }, + { + name: "chromium", + testIgnore: /responsive\.spec\.ts/, + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "firefox", + testIgnore: /responsive\.spec\.ts/, + use: { ...devices["Desktop Firefox"] }, + }, + { + name: "webkit", + testIgnore: /responsive\.spec\.ts/, + use: { ...devices["Desktop Safari"] }, + }, + { + name: "mobile-chromium", + testMatch: /responsive\.spec\.ts/, + use: { ...devices["Pixel 5"] }, + }, ], }); diff --git a/public/CHANGELOG.md b/public/CHANGELOG.md index 4192d88..1131662 100644 --- a/public/CHANGELOG.md +++ b/public/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.2.0 - 2026-09-02 + +- Added configurable multiline assembly, multi-file temporal merge, trace/span + and request-ID correlation with bounded retained evidence. +- Added OTLP/JSON output and a complete second streaming pass for filtered, + redacted export, with direct file-system writing where available and a + bounded Blob fallback. + ## 0.1.0 - 2026-09-01 - Added genuinely incremental `File.stream()` / `TextDecoder` scanning with progress, cancellation, line framing, and bounded memory. diff --git a/public/LICENSES/npm-runtime-licenses.txt b/public/LICENSES/npm-runtime-licenses.txt index 45a7c95..4f412f1 100644 --- a/public/LICENSES/npm-runtime-licenses.txt +++ b/public/LICENSES/npm-runtime-licenses.txt @@ -1,5 +1,5 @@ ============================================================================== -@add-ideas/toolbox-contract@0.2.3 +@add-ideas/toolbox-contract@0.3.0 Declared licence: Apache-2.0 ============================================================================== --- LICENSE --- @@ -198,7 +198,7 @@ Declared licence: Apache-2.0 ============================================================================== -@add-ideas/toolbox-helpers@0.1.0 +@add-ideas/toolbox-helpers@0.2.0 Declared licence: GPL-3.0-or-later ============================================================================== --- LICENSE --- @@ -879,7 +879,7 @@ Public License instead of this License. But first, please read ============================================================================== -@add-ideas/toolbox-shell-react@0.2.3 +@add-ideas/toolbox-shell-react@0.3.0 Declared licence: Apache-2.0 ============================================================================== --- LICENSE --- diff --git a/public/README.md b/public/README.md index 2a97801..960eb9b 100644 --- a/public/README.md +++ b/public/README.md @@ -8,13 +8,16 @@ Log Tools is a standalone local-first application in the [add·ideas Toolbox](ht - Incremental UTF-8 file scanning with byte/line progress, cancellation, a five-million-line stop, and an eight-GiB input gate - Bounded pasted/sample fallback and automatic or explicit plain, JSON Lines, nginx common/combined, syslog, and logfmt parsing +- Configurable streaming assembly for Java, .NET, Python, and generic indented multiline records, retaining their physical start/end lines - Normalized level, timestamp, source, message, and common structured fields with malformed-line diagnostics - Whole-scan bounded level/source/hour aggregation plus a 10,000-record retained preview for literal search and field filters +- Multi-file temporal merging for up to 16 separately streamed logs, with first-event offsets, backward-time diagnostics, W3C trace/span and common request-ID grouping, cross-file duration/source summaries, and retained-evidence caveats +- Focused OTLP/JSON log export with nanosecond timestamp strings, severity mapping, string attributes, trace/span context, and no invented timestamp for undated events - Deterministic email, IP, UUID, credential/query-secret, long-number, and exact-literal redaction recipes with generic or salted correlation tokens -- Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, with row/text caps and spreadsheet-safe CSV cells +- Filtered CSV, normalized NDJSON, or original-line text export from the retained preview, plus a complete second streaming pass with cancellation and progress; supported browsers write directly to a selected file and others use a 256 MiB Blob fallback - Inert ANSI/HTML handling, responsive Toolbox shell themes, nested-path offline PWA support, and deterministic release archives -The preview retains up to the first 10,000 nonblank parsed or malformed lines and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Search, redaction preview, and export therefore cannot include later records dropped from the preview. Auto-detection is heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, JSONL flattening is capped, and multiline stack traces remain separate plain records. +The preview retains up to the first 10,000 nonblank parsed or malformed records and stops earlier at a 32 MiB retained-character budget; counts and timeline buckets continue across the scanned portion. Multi-file mode retains at most 3,000 records per file (50,000/128 MiB across the analysis, 20,000 merged table events), so missing parent spans mean “not present in retained logs,” not necessarily a broken trace. OTLP/JSON export is capped at 20,000 records/64 MiB of retained text. Interactive search and redaction preview operate on bounded sets, while complete single-source export reapplies the same filters and recipes during a second pass over every scanned physical line. Auto-detection and automatic multiline classification are heuristic. The syslog parser covers RFC 5424 and a common RFC 3164 shape, nginx assumes the standard common/combined field order, and JSONL flattening is capped. See [Architecture](docs/ARCHITECTURE.md) and [Privacy and security](docs/PRIVACY-SECURITY.md) for exact bounds and trust assumptions. @@ -30,7 +33,7 @@ npm run test:browser ## Release -`npm run release:artifact` creates deterministic `release/log-tools-0.1.0.zip` and checksum files. +`npm run release:artifact` creates deterministic `release/log-tools-0.2.0.zip` and checksum files. ## Licence diff --git a/public/SOURCE.md b/public/SOURCE.md index 343ab6c..311c15a 100644 --- a/public/SOURCE.md +++ b/public/SOURCE.md @@ -1,7 +1,7 @@ # Corresponding source -The corresponding source for Log Tools 0.1.0 is available at: +The corresponding source for Log Tools 0.2.0 is available at: -https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.1.0 +https://git.add-ideas.de/lotobo/log-tools/src/tag/v0.2.0 Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`. diff --git a/public/THIRD_PARTY_NOTICES.md b/public/THIRD_PARTY_NOTICES.md index f52e239..8846877 100644 --- a/public/THIRD_PARTY_NOTICES.md +++ b/public/THIRD_PARTY_NOTICES.md @@ -1,12 +1,12 @@ # Third-party notices -Log Tools 0.1.0 directly depends on these runtime packages: +Log Tools 0.2.0 directly depends on these runtime packages: | Package | Pinned version | Declared licence | | -------------------------------- | -------------: | ---------------- | -| `@add-ideas/toolbox-contract` | 0.2.3 | Apache-2.0 | -| `@add-ideas/toolbox-helpers` | 0.1.0 | GPL-3.0-or-later | -| `@add-ideas/toolbox-shell-react` | 0.2.3 | Apache-2.0 | +| `@add-ideas/toolbox-contract` | 0.3.0 | Apache-2.0 | +| `@add-ideas/toolbox-helpers` | 0.2.0 | GPL-3.0-or-later | +| `@add-ideas/toolbox-shell-react` | 0.3.0 | Apache-2.0 | | `react` | 19.2.8 | MIT | | `react-dom` | 19.2.8 | MIT | diff --git a/public/docs/ARCHITECTURE.md b/public/docs/ARCHITECTURE.md index 2c1f7be..f8aaae9 100644 --- a/public/docs/ARCHITECTURE.md +++ b/public/docs/ARCHITECTURE.md @@ -1,9 +1,11 @@ # Architecture -Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string. +Log Tools is a static React/Vite application inside the shared Toolbox shell. `src/log/model.ts` contains the streaming and bounded trust boundary. `scanLogBlob()` obtains a `ReadableStream` from the supplied `Blob`/`File`, reads byte chunks, decodes UTF-8 incrementally with `TextDecoder.decode(..., {stream:true})`, and feeds a line framer that retains at most 256 KiB for one physical line. A bounded multiline assembler groups configured stack-trace continuations and records their first and last physical lines. It never calls `File.text()`, `arrayBuffer()`, or constructs a complete-file string. Automatic detection buffers only the first 30 nonblank bounded lines, chooses among JSON Lines, nginx common/combined, RFC 5424/common RFC 3164 syslog, logfmt, and plain text, then replays that small buffer through the selected parser. Explicit selection skips detection. Parsing continues up to five million lines or eight GiB. At most the first 10,000 nonblank records and 32 MiB of retained characters are kept; individual retained raw/message strings, fields, level/source maps, hourly timestamp buckets, field names, counters, and progress are independently capped. Reads yield between progress intervals so cancellation and UI updates can run. -Filtering operates on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Exports use the filtered retained preview, at most 50,000 rows and 10 MiB of text. CSV prefixes spreadsheet-formula-leading cells. +Filtering operates interactively on the retained preview. Correlation counts describe the scanned portion. Redaction recipes transform raw text, normalized messages, sources, and extracted values using fixed, linear-oriented patterns and an optional exact literal. Expanded redaction output is capped per value and truncation is reported. Salted pseudonyms use a stable non-cryptographic hash: useful for local equality correlation, not encryption or resistance to guessing. Quick exports use the preview; `exportLogBlob()` performs a second cancellable streaming pass and reapplies parsing, filters, and redaction to all records. It writes through a supplied `WritableStream`, or returns a Blob under a 256 MiB cap. CSV prefixes spreadsheet-formula-leading cells. + +Multi-file analysis streams each selected file through the same scanner, retains at most 3,000 records per file, and merges only those retained records. `analyzeLogSources()` caps file count, record count, retained characters, merged events, and correlation groups. It recognizes valid non-zero W3C-sized trace/span IDs plus bounded common request identifiers, reports unresolved parent references as retained-evidence gaps, and never adjusts timestamps. `exportOtlpJson()` produces a focused bounded OTLP/JSON request from retained records; it does not claim to reconstruct complete traces or export the unretained source. React renders log values through text nodes only. ANSI escape sequences are removed and other control bytes are replaced; HTML-like content is never interpreted. Relative assets and a same-origin scoped service worker keep the build relocatable below nested portal routes. There is no telemetry, database, server API, worker, or remote dependency at runtime. diff --git a/public/sw.js b/public/sw.js index b556a6d..dfcd08e 100644 --- a/public/sw.js +++ b/public/sw.js @@ -1,5 +1,5 @@ const CACHE_PREFIX = "log-tools-shell-"; -const CACHE_NAME = CACHE_PREFIX + "0.1.0"; +const CACHE_NAME = CACHE_PREFIX + "0.2.0"; const CORE = ["./", "./manifest.webmanifest", "./favicon.svg"]; self.addEventListener("install", (event) => { event.waitUntil( diff --git a/public/toolbox-app.json b/public/toolbox-app.json index 38f8080..b3ef3cd 100644 --- a/public/toolbox-app.json +++ b/public/toolbox-app.json @@ -3,7 +3,7 @@ "schemaVersion": 1, "id": "de.add-ideas.log-tools", "name": "Log Tools", - "version": "0.1.0", + "version": "0.2.0", "description": "Stream, inspect, correlate, redact, and export logs locally.", "entry": "./", "icon": "./favicon.svg", @@ -21,6 +21,23 @@ "crossOriginIsolated": false, "topLevelContext": false }, + "io": { + "accepts": [ + { "mediaType": "text/plain", "extensions": [".log", ".txt"] }, + { + "mediaType": "application/x-ndjson", + "extensions": [".ndjson", ".jsonl"] + }, + { "mediaType": "application/json", "extensions": [".json"] } + ], + "produces": [ + { "mediaType": "text/plain", "extensions": [".log", ".txt"] }, + { "mediaType": "application/x-ndjson", "extensions": [".ndjson"] }, + { "mediaType": "text/csv", "extensions": [".csv"] }, + { "mediaType": "application/json", "extensions": [".json"] } + ] + }, + "capabilities": { "required": [], "optional": ["file-system-access"] }, "privacy": { "processing": "local", "fileUploads": true, diff --git a/src/components/HelpDialog.tsx b/src/components/HelpDialog.tsx index 7cab2c8..d75fd9b 100644 --- a/src/components/HelpDialog.tsx +++ b/src/components/HelpDialog.tsx @@ -35,6 +35,12 @@ export function HelpDialog({ Stream large local log files, parse common formats, search the retained preview, correlate bounded aggregates, redact identifiers, and export.

+

+ Multi-file mode merges bounded per-file previews by timestamp and groups + valid trace/span or request identifiers. It reports evidence gaps rather + than assuming a complete distributed trace, and can map the retained + records to a focused OTLP/JSON payload. +

All processing is performed in this browser. Imported data is treated as untrusted and bounded before parsing. ANSI and HTML remain inert text, diff --git a/src/components/Workbench.tsx b/src/components/Workbench.tsx index 0f0a5ad..011aec7 100644 --- a/src/components/Workbench.tsx +++ b/src/components/Workbench.tsx @@ -3,14 +3,21 @@ import { triggerBlobDownload } from "@add-ideas/toolbox-helpers"; import { LOG_LIMITS, ScanCancelledError, + analyzeLogSources, + exportLogBlob, + exportOtlpJson, exportRecords, filterRecords, redactRecords, scanLogBlob, scanPastedLog, type ExportResult, + type FullExportProgress, + type FullExportResult, type LogFilters, type LogFormat, + type MultilineMode, + type NamedLogSource, type RedactionOptions, type ScanProgress, type ScanReport, @@ -30,7 +37,15 @@ const formatLabels: Record = { syslog: "Syslog (RFC 5424 / common RFC 3164)", logfmt: "logfmt", }; -type Tab = "explore" | "correlate" | "redact" | "export"; +const multilineLabels: Record = { + off: "One event per physical line", + auto: "Auto: Java, .NET, and Python traces", + java: "Java stack traces", + dotnet: ".NET stack traces", + python: "Python tracebacks", + indented: "Every indented continuation", +}; +type Tab = "explore" | "correlate" | "traces" | "redact" | "export"; const defaultRedaction: RedactionOptions = { email: true, ip: true, @@ -62,7 +77,9 @@ function save(result: ExportResult) { export function Workbench() { const [tab, setTab] = useState("explore"); const [format, setFormat] = useState("auto"); + const [multiline, setMultiline] = useState("auto"); const [file, setFile] = useState(); + const [sourceBlob, setSourceBlob] = useState(); const [paste, setPaste] = useState(sample); const [report, setReport] = useState(); const [progress, setProgress] = useState(); @@ -79,7 +96,15 @@ export function Workbench() { ); const [redactExport, setRedactExport] = useState(true); const [lastExport, setLastExport] = useState(); + const [lastFullExport, setLastFullExport] = useState(); + const [multiReports, setMultiReports] = useState([]); + const [traceScanning, setTraceScanning] = useState(false); + const [traceProgress, setTraceProgress] = useState(""); + const [exporting, setExporting] = useState(false); + const [exportProgress, setExportProgress] = useState(); const controller = useRef(undefined); + const exportController = useRef(undefined); + const traceController = useRef(undefined); const filtered = useMemo( () => filterRecords(report?.preview ?? [], filters), @@ -91,6 +116,22 @@ export function Workbench() { ); const levels = report?.levelCounts.map((item) => item.key) ?? []; const sources = report?.sourceCounts.map((item) => item.key) ?? []; + const traceSources = useMemo( + () => + multiReports.length + ? multiReports.map((item) => ({ + filename: item.filename, + records: item.preview, + })) + : report + ? [{ filename: report.filename, records: report.preview }] + : [], + [multiReports, report], + ); + const traceAnalysis = useMemo( + () => analyzeLogSources(traceSources), + [traceSources], + ); const scan = async (blob: Blob, filename: string, pasted = false) => { controller.current?.abort(); @@ -104,18 +145,19 @@ export function Workbench() { const next = pasted ? await scanPastedLog( paste, - { format, filename }, + { format, filename, multiline }, setProgress, nextController.signal, ) : await scanLogBlob( blob, - { format, filename }, + { format, filename, multiline }, setProgress, nextController.signal, ); if (controller.current !== nextController) return; setReport(next); + setSourceBlob(blob); setFilters({}); setLastExport(undefined); setStatus( @@ -147,19 +189,135 @@ export function Workbench() { }; const clear = () => { controller.current?.abort(); + exportController.current?.abort(); + traceController.current?.abort(); setFile(undefined); setPaste(""); setReport(undefined); + setSourceBlob(undefined); setProgress(undefined); + setMultiReports([]); + setTraceProgress(""); setError(""); setStatus("Workspace cleared from application memory."); }; + const scanTraceFiles = async (files: FileList | null) => { + if (!files?.length) return; + if (files.length > 16) { + setError("Multi-file analysis accepts at most 16 files at once."); + return; + } + traceController.current?.abort(); + const nextController = new AbortController(); + traceController.current = nextController; + setTraceScanning(true); + setError(""); + const next: ScanReport[] = []; + try { + for (const [index, selected] of [...files].entries()) { + setTraceProgress( + `Scanning ${index + 1} of ${files.length}: ${selected.name}`, + ); + const scanned = await scanLogBlob( + selected, + { + format, + multiline, + filename: selected.webkitRelativePath || selected.name, + previewLimit: 3_000, + }, + (value) => + setTraceProgress( + `${index + 1} of ${files.length}: ${selected.name} · ${bytes(value.bytesRead)} of ${bytes(value.totalBytes)}`, + ), + nextController.signal, + ); + next.push(scanned); + } + if (traceController.current !== nextController) return; + setMultiReports(next); + setTraceProgress( + `${next.length} files scanned; correlation uses ${next.reduce((sum, item) => sum + item.preview.length, 0).toLocaleString()} bounded preview records.`, + ); + } catch (reason) { + if (traceController.current !== nextController) return; + if (reason instanceof ScanCancelledError) + setTraceProgress( + "Multi-file scan cancelled; the previous analysis remains.", + ); + else setError(message(reason, "Could not scan the log set.")); + } finally { + if (traceController.current === nextController) { + setTraceScanning(false); + traceController.current = undefined; + } + } + }; const prepareExport = () => { const records = redactExport ? redacted.records : filtered; const result = exportRecords(records, exportFormat); setLastExport(result); save(result); }; + const exportCompleteSource = async () => { + if (!report || !sourceBlob) return; + exportController.current?.abort(); + const nextController = new AbortController(); + exportController.current = nextController; + setExporting(true); + setExportProgress({ + bytesRead: 0, + totalBytes: sourceBlob.size, + physicalLines: 0, + rows: 0, + bytesWritten: 0, + }); + setError(""); + try { + const picker = ( + window as Window & { + showSaveFilePicker?: (options: { suggestedName: string }) => Promise<{ + createWritable(): Promise>; + }>; + } + ).showSaveFilePicker; + let writable: WritableStream | undefined; + if (picker) { + const handle = await picker({ + suggestedName: `filtered-logs.${exportFormat === "text" ? "log" : exportFormat}`, + }); + writable = await handle.createWritable(); + } + const result = await exportLogBlob( + sourceBlob, + { + format: exportFormat, + inputFormat: report.format, + filters, + ...(redactExport ? { redaction } : {}), + multiline, + ...(writable ? { writable } : {}), + }, + setExportProgress, + nextController.signal, + ); + if (exportController.current !== nextController) return; + if (result.blob) { + triggerBlobDownload(result.blob, `filtered-logs.${result.extension}`); + } + setLastFullExport(result); + } catch (reason) { + if (exportController.current !== nextController) return; + if (reason instanceof DOMException && reason.name === "AbortError") + return; + setError(message(reason, "Could not export the complete source.")); + } finally { + if (exportController.current === nextController) { + setExporting(false); + exportController.current = undefined; + } + } + }; const progressValue = progress?.totalBytes ? Math.min(100, (progress.bytesRead / progress.totalBytes) * 100) : 0; @@ -205,6 +363,22 @@ export function Workbench() { ))} +

+
{tab === "explore" && ( -
+