diff --git a/.gitea/workflows/verify.yml b/.gitea/workflows/verify.yml
new file mode 100644
index 0000000..b84a7cb
--- /dev/null
+++ b/.gitea/workflows/verify.yml
@@ -0,0 +1,39 @@
+name: Verify
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+ workflow_dispatch:
+
+concurrency:
+ group: verify-${{ gitea.repository }}-${{ gitea.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ verify:
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ env:
+ CI: "true"
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ cache: npm
+ - name: Select declared npm version
+ run: npm install --global npm@11.17.0
+ - name: Install dependencies
+ run: npm ci
+ - name: Audit runtime dependencies
+ run: npm audit --omit=dev --audit-level=moderate
+ - name: Check, test, and build
+ run: npm run check
+ - name: Install browser engines
+ run: npx playwright install --with-deps chromium firefox webkit
+ - name: Browser tests
+ run: npm run test:browser
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9c2ba54..b3b995a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
# Changelog
+## 0.2.0 - 2026-09-02
+
+- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue,
+ explicit schema/loss evidence, cancellable worker execution and reusable
+ recipe/export handoffs.
+
## 0.1.0 - 2026-09-01
- Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export.
diff --git a/README.md b/README.md
index 8b3d5dc..8cee735 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,14 @@
# Flow Tools
-Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage.
+Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted
+or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map,
+sort, group, join, type-format, deduplicate, slice and explode primitives. Every
+stage has an inspectable snapshot, dataset analysis and explicit loss notes.
+Recipes are deterministic validated JSON; there is no arbitrary code, eval,
+plugin or network stage.
-Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`.
+Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30
+stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result
+JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`,
+`npm run check`, `npm run test:browser`, then
+`npm run package:release -- --force`.
diff --git a/SOURCE.md b/SOURCE.md
index c66081b..ebee4a3 100644
--- a/SOURCE.md
+++ b/SOURCE.md
@@ -1,3 +1,3 @@
# Source identity
-Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
+Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index b03e91e..4de531c 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -1,3 +1,5 @@
# Third-party notices
-Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets.
+Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox
+Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0
+(GPL-3.0-or-later). Exact texts are generated in release assets.
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 69124d3..8abd343 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -1,3 +1,12 @@
# Architecture
-The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run.
+The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and
+produces JSON values. The interpreter copies rows stage-by-stage and accepts a
+fixed discriminated grammar, including deduplication, bounded slicing and array
+explosion. It records output, loss notes and dataset statistics after each run.
+
+Analysis and simulation run in a disposable module worker. Progress messages
+are revision-checked, cancellation terminates the worker, and React retains the
+last successful snapshots. Recipe imports are size-bounded and validate every
+field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline
+SVG are generated entirely in the browser.
diff --git a/package-lock.json b/package-lock.json
index 7dc5db1..2243f52 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,23 +1,23 @@
{
"name": "flow-tools",
- "version": "0.1.0",
+ "version": "0.2.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "flow-tools",
- "version": "0.1.0",
+ "version": "0.2.0",
"license": "GPL-3.0-or-later",
"dependencies": {
- "@add-ideas/toolbox-contract": "0.2.3",
- "@add-ideas/toolbox-helpers": "0.1.0",
- "@add-ideas/toolbox-shell-react": "0.2.3",
+ "@add-ideas/toolbox-contract": "0.3.0",
+ "@add-ideas/toolbox-helpers": "0.2.0",
+ "@add-ideas/toolbox-shell-react": "0.3.0",
"@xmldom/xmldom": "0.9.12",
"react": "19.2.8",
"react-dom": "19.2.8"
},
"devDependencies": {
- "@add-ideas/toolbox-testkit": "0.2.3",
+ "@add-ideas/toolbox-testkit": "0.3.0",
"@eslint/js": "10.0.1",
"@playwright/test": "1.62.1",
"@testing-library/jest-dom": "6.9.1",
@@ -43,24 +43,24 @@
}
},
"node_modules/@add-ideas/toolbox-contract": {
- "version": "0.2.3",
- "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.2.3/toolbox-contract-0.2.3.tgz",
- "integrity": "sha512-T0PVSuMT40GjTDfQJhEEY3ZawQq8zz1/ry95JdKI6W39CdLacaRXdGnEpDCMHt+jUbf1Jz7Nat/M5dFCgKVM9A==",
+ "version": "0.3.0",
+ "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz",
+ "integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==",
"license": "Apache-2.0"
},
"node_modules/@add-ideas/toolbox-helpers": {
- "version": "0.1.0",
- "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.1.0/toolbox-helpers-0.1.0.tgz",
- "integrity": "sha512-UKl1Oxekedf8D2df86VrnVA53AcMhrnh6iUPXY+k8frirBXotb0yd8SGT+IF/3hcqYwcYe/v9WVFuSgKtIYVnw==",
+ "version": "0.2.0",
+ "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.2.0/toolbox-helpers-0.2.0.tgz",
+ "integrity": "sha512-SdOqkw+P+3J3fa5iVkzb5P15rVepB001GNV21Oh8w0CZcVL+YRltgD/s+MVcTyrNijWQf3E5vtQON/3N2LLyKg==",
"license": "GPL-3.0-or-later"
},
"node_modules/@add-ideas/toolbox-shell-react": {
- "version": "0.2.3",
- "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.2.3/toolbox-shell-react-0.2.3.tgz",
- "integrity": "sha512-DT5lQDH48BFkFcmFLZnQh7+Cm73JzBPcmp5WzUXypfkUXpEyDYHzaXgmW4kZ0edSwh4RK4sPmx+JPtK0X4aKCQ==",
+ "version": "0.3.0",
+ "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz",
+ "integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==",
"license": "Apache-2.0",
"dependencies": {
- "@add-ideas/toolbox-contract": "0.2.3"
+ "@add-ideas/toolbox-contract": "0.3.0"
},
"peerDependencies": {
"react": ">=18 <20",
@@ -68,13 +68,13 @@
}
},
"node_modules/@add-ideas/toolbox-testkit": {
- "version": "0.2.3",
- "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.2.3/toolbox-testkit-0.2.3.tgz",
- "integrity": "sha512-sq1MwhKWfFKen+N+124hl74qQimRSvmQ9sOU7jdcI+2qCKZ67+2B8rWyezeV80uTFu4Jv6deHksfYQ/tKNV6XQ==",
+ "version": "0.3.0",
+ "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz",
+ "integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
- "@add-ideas/toolbox-contract": "0.2.3"
+ "@add-ideas/toolbox-contract": "0.3.0"
},
"bin": {
"toolbox-check": "dist/cli.js"
diff --git a/package.json b/package.json
index 3127f09..c9b4507 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "flow-tools",
- "version": "0.1.0",
+ "version": "0.2.0",
"description": "Build and inspect bounded local data transformation pipelines.",
"license": "GPL-3.0-or-later",
"author": "Albrecht Degering",
@@ -9,6 +9,9 @@
"url": "git+https://git.add-ideas.de/lotobo/flow-tools.git"
},
"homepage": "https://git.add-ideas.de/lotobo/flow-tools",
+ "bugs": {
+ "url": "https://git.add-ideas.de/lotobo/flow-tools/issues"
+ },
"private": true,
"type": "module",
"engines": {
@@ -34,15 +37,15 @@
"release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force"
},
"dependencies": {
- "@add-ideas/toolbox-contract": "0.2.3",
- "@add-ideas/toolbox-helpers": "0.1.0",
- "@add-ideas/toolbox-shell-react": "0.2.3",
+ "@add-ideas/toolbox-contract": "0.3.0",
+ "@add-ideas/toolbox-helpers": "0.2.0",
+ "@add-ideas/toolbox-shell-react": "0.3.0",
"@xmldom/xmldom": "0.9.12",
"react": "19.2.8",
"react-dom": "19.2.8"
},
"devDependencies": {
- "@add-ideas/toolbox-testkit": "0.2.3",
+ "@add-ideas/toolbox-testkit": "0.3.0",
"@eslint/js": "10.0.1",
"@playwright/test": "1.62.1",
"@testing-library/jest-dom": "6.9.1",
diff --git a/playwright.config.ts b/playwright.config.ts
index 74bc44f..af8741f 100644
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -12,7 +12,25 @@ export default defineConfig({
timeout: 180000,
},
projects: [
- { name: "chromium", use: { ...devices["Desktop Chrome"] } },
- { name: "firefox", use: { ...devices["Desktop Firefox"] } },
+ {
+ name: "chromium",
+ testIgnore: /responsive\.spec\.ts/,
+ use: { ...devices["Desktop Chrome"] },
+ },
+ {
+ name: "firefox",
+ testIgnore: /responsive\.spec\.ts/,
+ use: { ...devices["Desktop Firefox"] },
+ },
+ {
+ name: "webkit",
+ testIgnore: /responsive\.spec\.ts/,
+ use: { ...devices["Desktop Safari"] },
+ },
+ {
+ name: "mobile-chromium",
+ testMatch: /responsive\.spec\.ts/,
+ use: { ...devices["Pixel 5"] },
+ },
],
});
diff --git a/public/CHANGELOG.md b/public/CHANGELOG.md
index 9c2ba54..b3b995a 100644
--- a/public/CHANGELOG.md
+++ b/public/CHANGELOG.md
@@ -1,5 +1,11 @@
# Changelog
+## 0.2.0 - 2026-09-02
+
+- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue,
+ explicit schema/loss evidence, cancellable worker execution and reusable
+ recipe/export handoffs.
+
## 0.1.0 - 2026-09-01
- Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export.
diff --git a/public/LICENSES/npm-runtime-licenses.txt b/public/LICENSES/npm-runtime-licenses.txt
index 647441c..c1c48cd 100644
--- a/public/LICENSES/npm-runtime-licenses.txt
+++ b/public/LICENSES/npm-runtime-licenses.txt
@@ -1,5 +1,5 @@
==============================================================================
-@add-ideas/toolbox-contract@0.2.3
+@add-ideas/toolbox-contract@0.3.0
Declared licence: Apache-2.0
==============================================================================
--- LICENSE ---
@@ -198,7 +198,7 @@ Declared licence: Apache-2.0
==============================================================================
-@add-ideas/toolbox-helpers@0.1.0
+@add-ideas/toolbox-helpers@0.2.0
Declared licence: GPL-3.0-or-later
==============================================================================
--- LICENSE ---
@@ -879,7 +879,7 @@ Public License instead of this License. But first, please read
==============================================================================
-@add-ideas/toolbox-shell-react@0.2.3
+@add-ideas/toolbox-shell-react@0.3.0
Declared licence: Apache-2.0
==============================================================================
--- LICENSE ---
diff --git a/public/README.md b/public/README.md
index 8b3d5dc..8cee735 100644
--- a/public/README.md
+++ b/public/README.md
@@ -1,5 +1,14 @@
# Flow Tools
-Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage.
+Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted
+or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map,
+sort, group, join, type-format, deduplicate, slice and explode primitives. Every
+stage has an inspectable snapshot, dataset analysis and explicit loss notes.
+Recipes are deterministic validated JSON; there is no arbitrary code, eval,
+plugin or network stage.
-Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`.
+Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30
+stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result
+JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`,
+`npm run check`, `npm run test:browser`, then
+`npm run package:release -- --force`.
diff --git a/public/SOURCE.md b/public/SOURCE.md
index c66081b..ebee4a3 100644
--- a/public/SOURCE.md
+++ b/public/SOURCE.md
@@ -1,3 +1,3 @@
# Source identity
-Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
+Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
diff --git a/public/THIRD_PARTY_NOTICES.md b/public/THIRD_PARTY_NOTICES.md
index b03e91e..4de531c 100644
--- a/public/THIRD_PARTY_NOTICES.md
+++ b/public/THIRD_PARTY_NOTICES.md
@@ -1,3 +1,5 @@
# Third-party notices
-Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets.
+Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox
+Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0
+(GPL-3.0-or-later). Exact texts are generated in release assets.
diff --git a/public/docs/ARCHITECTURE.md b/public/docs/ARCHITECTURE.md
index 69124d3..8abd343 100644
--- a/public/docs/ARCHITECTURE.md
+++ b/public/docs/ARCHITECTURE.md
@@ -1,3 +1,12 @@
# Architecture
-The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run.
+The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and
+produces JSON values. The interpreter copies rows stage-by-stage and accepts a
+fixed discriminated grammar, including deduplication, bounded slicing and array
+explosion. It records output, loss notes and dataset statistics after each run.
+
+Analysis and simulation run in a disposable module worker. Progress messages
+are revision-checked, cancellation terminates the worker, and React retains the
+last successful snapshots. Recipe imports are size-bounded and validate every
+field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline
+SVG are generated entirely in the browser.
diff --git a/public/sw.js b/public/sw.js
index 8463933..d4da2e2 100644
--- a/public/sw.js
+++ b/public/sw.js
@@ -1,4 +1,4 @@
-const CACHE = "flow-tools-v0.1.0",
+const CACHE = "flow-tools-v0.2.0",
APP = [
"./",
"./index.html",
diff --git a/public/toolbox-app.json b/public/toolbox-app.json
index 6e7178f..9c8663a 100644
--- a/public/toolbox-app.json
+++ b/public/toolbox-app.json
@@ -3,7 +3,7 @@
"schemaVersion": 1,
"id": "de.add-ideas.flow-tools",
"name": "Flow Tools",
- "version": "0.1.0",
+ "version": "0.2.0",
"description": "Transform structured data locally.",
"entry": "./",
"icon": "./favicon.svg",
@@ -16,11 +16,28 @@
},
"requirements": {
"secureContext": false,
- "workers": false,
+ "workers": true,
"indexedDb": false,
"crossOriginIsolated": false,
"topLevelContext": false
},
+ "io": {
+ "accepts": [
+ { "mediaType": "application/json", "extensions": [".json"] },
+ {
+ "mediaType": "application/x-ndjson",
+ "extensions": [".ndjson", ".jsonl"]
+ },
+ { "mediaType": "text/csv", "extensions": [".csv"] },
+ { "mediaType": "application/xml", "extensions": [".xml"] }
+ ],
+ "produces": [
+ { "mediaType": "application/json", "extensions": [".json"] },
+ { "mediaType": "text/csv", "extensions": [".csv"] },
+ { "mediaType": "image/svg+xml", "extensions": [".svg"] }
+ ]
+ },
+ "capabilities": { "required": ["workers"], "optional": [] },
"privacy": {
"processing": "local",
"fileUploads": true,
diff --git a/src/components/HelpDialog.tsx b/src/components/HelpDialog.tsx
index 3518f9d..ac65c6e 100644
--- a/src/components/HelpDialog.tsx
+++ b/src/components/HelpDialog.tsx
@@ -18,7 +18,12 @@ export function HelpDialog({
onCancel={onClose}
aria-label="About Flow Tools"
>
-
-
- Run complete pipeline
-
+
+ void run()}
+ disabled={busy}
+ >
+ Run complete pipeline
+
+ {busy && (
+ operation.current.controller?.abort()}>
+ Cancel
+
+ )}
+
{status}
+ {busy && (
+
+ )}
@@ -196,6 +305,9 @@ export function Workbench() {
+
+
+
)}
+
+
+
- Rows
+ -
+ {result.analysis.inputRows} → {result.analysis.outputRows}
+
+
+
+
- Fields
+ - {result.analysis.fields}
+
+
+
- Empty cells
+ - {result.analysis.nullCells}
+
+
+
- Input duplicates
+ - {result.analysis.duplicateRows}
+
+
download("json")}>Result JSON
download("csv")}>Result CSV
download("recipe")}>Recipe JSON
+ download("svg")}>Pipeline SVG
Import deterministic recipe
@@ -263,6 +396,14 @@ export function Workbench() {
aria-label="Recipe JSON"
/>
Validate and import
+
@@ -303,8 +444,20 @@ function StageEditor({
@@ -415,6 +568,36 @@ function Config({
/>
>
);
+ if (stage.type === "deduplicate")
+ return (
+ set("fields", v)}
+ />
+ );
+ if (stage.type === "slice")
+ return (
+ <>
+ set("offset", v)} />
+ set("count", v)} />
+ >
+ );
+ if (stage.type === "explode")
+ return (
+ <>
+ set("field", v)}
+ />
+ set("target", v)}
+ />
+ >
+ );
return (
<>
set("target", v)} />
@@ -530,5 +713,26 @@ function defaults(t: StageType): Record {
mode: "left",
prefix: "right_",
}
- : { field: "score", as: "number" };
+ : t === "format"
+ ? { field: "score", as: "number" }
+ : t === "deduplicate"
+ ? { fields: "" }
+ : t === "slice"
+ ? { offset: "0", count: "100" }
+ : { field: "items", target: "item" };
+}
+
+function inferFormat(fileName: string): DataFormat {
+ const extension = fileName.split(".").at(-1)?.toLowerCase();
+ if (extension === "csv") return "csv";
+ if (extension === "ndjson" || extension === "jsonl") return "ndjson";
+ if (extension === "xml") return "xml";
+ return "json";
+}
+
+function nextStageId(stages: readonly Stage[]): string {
+ const ids = new Set(stages.map((stage) => stage.id));
+ let suffix = 1;
+ while (ids.has(`stage-${suffix}`)) suffix += 1;
+ return `stage-${suffix}`;
}
diff --git a/src/core/flow-client.ts b/src/core/flow-client.ts
new file mode 100644
index 0000000..a24104d
--- /dev/null
+++ b/src/core/flow-client.ts
@@ -0,0 +1,57 @@
+import type { JsonValue } from "@add-ideas/toolbox-helpers";
+import {
+ runPipeline,
+ type PipelineProgress,
+ type PipelineResult,
+ type Stage,
+} from "./pipeline";
+
+export async function runFlowJob(
+ input: Record[],
+ stages: Stage[],
+ right: Record[],
+ options: {
+ signal?: AbortSignal;
+ onProgress?: (progress: PipelineProgress) => void;
+ } = {},
+): Promise {
+ if (options.signal?.aborted) throw abortError();
+ if (typeof Worker === "undefined")
+ return runPipeline(input, stages, right, options.onProgress);
+ const worker = new Worker(new URL("./flow.worker.ts", import.meta.url), {
+ type: "module",
+ name: "flow-tools-analysis",
+ });
+ const id = Date.now() + Math.random();
+ return await new Promise((resolve, reject) => {
+ const cleanup = () => {
+ options.signal?.removeEventListener("abort", cancel);
+ worker.terminate();
+ };
+ const cancel = () => {
+ cleanup();
+ reject(abortError());
+ };
+ options.signal?.addEventListener("abort", cancel, { once: true });
+ worker.addEventListener("message", (event) => {
+ if (event.data?.id !== id) return;
+ if (event.data.type === "progress") {
+ options.onProgress?.(event.data.progress as PipelineProgress);
+ return;
+ }
+ cleanup();
+ if (event.data.type === "result")
+ resolve(event.data.result as PipelineResult);
+ else reject(new Error(String(event.data.error ?? "Flow worker failed.")));
+ });
+ worker.addEventListener("error", () => {
+ cleanup();
+ reject(new Error("The isolated flow worker stopped unexpectedly."));
+ });
+ worker.postMessage({ id, input, stages, right });
+ });
+}
+
+function abortError() {
+ return new DOMException("Flow operation cancelled.", "AbortError");
+}
diff --git a/src/core/flow.worker.ts b/src/core/flow.worker.ts
new file mode 100644
index 0000000..c7dc4b5
--- /dev/null
+++ b/src/core/flow.worker.ts
@@ -0,0 +1,31 @@
+///
+import { runPipeline, type PipelineProgress, type Stage } from "./pipeline";
+import type { JsonValue } from "@add-ideas/toolbox-helpers";
+
+interface Request {
+ id: number;
+ input: Record[];
+ right: Record[];
+ stages: Stage[];
+}
+
+const scope = self as DedicatedWorkerGlobalScope;
+scope.addEventListener("message", (event: MessageEvent) => {
+ const request = event.data;
+ try {
+ const result = runPipeline(
+ request.input,
+ request.stages,
+ request.right,
+ (progress: PipelineProgress) =>
+ scope.postMessage({ id: request.id, type: "progress", progress }),
+ );
+ scope.postMessage({ id: request.id, type: "result", result });
+ } catch (error) {
+ scope.postMessage({
+ id: request.id,
+ type: "error",
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+});
diff --git a/src/core/pipeline.ts b/src/core/pipeline.ts
index 23d52bc..e262589 100644
--- a/src/core/pipeline.ts
+++ b/src/core/pipeline.ts
@@ -5,7 +5,17 @@ import {
} from "@add-ideas/toolbox-helpers";
import { isObject } from "./data";
export type StageType =
- "select" | "rename" | "filter" | "map" | "sort" | "group" | "join" | "format";
+ | "select"
+ | "rename"
+ | "filter"
+ | "map"
+ | "sort"
+ | "group"
+ | "join"
+ | "format"
+ | "deduplicate"
+ | "slice"
+ | "explode";
export interface Stage {
id: string;
type: StageType;
@@ -22,6 +32,20 @@ export interface PipelineResult {
snapshots: Snapshot[];
rows: Record[];
recipe: string;
+ analysis: PipelineAnalysis;
+}
+export interface PipelineAnalysis {
+ inputRows: number;
+ outputRows: number;
+ fields: number;
+ nullCells: number;
+ duplicateRows: number;
+ stageRows: number[];
+}
+export interface PipelineProgress {
+ stage: string;
+ completed: number;
+ total: number;
}
const MAX_STAGES = 30,
MAX_ROWS = 10000,
@@ -36,11 +60,13 @@ export function runPipeline(
input: Record[],
stages: Stage[],
right: Record[] = [],
+ onProgress?: (progress: PipelineProgress) => void,
): PipelineResult {
if (stages.length > MAX_STAGES) throw new Error("Recipe exceeds 30 stages.");
let rows = input.map((row) => ({ ...row }));
const snapshots: Snapshot[] = [{ name: "Input", rows, losses: [] }];
- for (const stage of stages) {
+ onProgress?.({ stage: "Input analysis", completed: 0, total: stages.length });
+ for (const [index, stage] of stages.entries()) {
if (!stage.enabled) {
snapshots.push({
name: `${stage.type} (disabled)`,
@@ -48,6 +74,11 @@ export function runPipeline(
losses: [],
stage,
});
+ onProgress?.({
+ stage: `${stage.type} (disabled)`,
+ completed: index + 1,
+ total: stages.length,
+ });
continue;
}
const output = apply(rows, stage, right);
@@ -55,8 +86,18 @@ export function runPipeline(
if (rows.length > MAX_JOIN)
throw new Error(`Stage ${stage.id} expands beyond 20,000 rows.`);
snapshots.push({ name: stage.type, rows, losses: output.losses, stage });
+ onProgress?.({
+ stage: stage.type,
+ completed: index + 1,
+ total: stages.length,
+ });
}
- return { snapshots, rows, recipe: exportRecipe(stages) };
+ return {
+ snapshots,
+ rows,
+ recipe: exportRecipe(stages),
+ analysis: analyze(input, rows, snapshots),
+ };
}
function apply(
rows: Record[],
@@ -189,6 +230,54 @@ function apply(
losses,
};
}
+ if (stage.type === "deduplicate") {
+ const fields = list(c.fields);
+ const seen = new Set();
+ const output = rows.filter((row) => {
+ const key = stableStringify(
+ fields.length
+ ? Object.fromEntries(
+ fields.map((field) => [field, row[field] ?? null]),
+ )
+ : row,
+ );
+ if (seen.has(key)) return false;
+ seen.add(key);
+ return true;
+ });
+ return {
+ rows: output,
+ losses: [`${rows.length - output.length} duplicate rows were discarded.`],
+ };
+ }
+ if (stage.type === "slice") {
+ const offset = boundedInteger(c.offset ?? "0", "Slice offset", 0, MAX_JOIN),
+ count = boundedInteger(c.count ?? "100", "Slice count", 0, MAX_JOIN),
+ sliced = rows.slice(offset, offset + count);
+ return {
+ rows: sliced,
+ losses: [
+ `${rows.length - sliced.length} rows fall outside the selected slice.`,
+ ],
+ };
+ }
+ if (stage.type === "explode") {
+ const field = required(c.field, "Explode field"),
+ target = c.target?.trim() || field,
+ output: Record[] = [];
+ for (const row of rows) {
+ const value = row[field];
+ if (!Array.isArray(value)) {
+ output.push({ ...row, [target]: value ?? null });
+ continue;
+ }
+ for (const item of value) output.push({ ...row, [target]: item });
+ }
+ return {
+ rows: output,
+ losses: ["Array values were expanded into individual rows."],
+ };
+ }
if (stage.type === "group") {
const key = required(c.key, "Group key"),
fn = (c.fn ?? "count").toLowerCase(),
@@ -288,6 +377,7 @@ export function importRecipe(source: string): Stage[] {
throw new Error("Not a Flow Tools recipe v1.");
if (value.stages.length > MAX_STAGES)
throw new Error("Recipe exceeds 30 stages.");
+ const ids = new Set();
return value.stages.map((raw, index) => {
if (
!isObject(raw) ||
@@ -301,23 +391,93 @@ export function importRecipe(source: string): Stage[] {
"group",
"join",
"format",
+ "deduplicate",
+ "slice",
+ "explode",
].includes(raw.type) ||
!raw.config ||
!isObject(raw.config)
)
throw new Error(`Invalid recipe stage ${index + 1}.`);
+ if (raw.enabled !== undefined && typeof raw.enabled !== "boolean")
+ throw new Error(`Stage ${index + 1} enabled must be a boolean.`);
+ const id = typeof raw.id === "string" ? raw.id : `stage-${index + 1}`;
+ if (!/^[A-Za-z0-9_-]{1,64}$/u.test(id) || ids.has(id))
+ throw new Error(`Stage ${index + 1} needs a safe unique id.`);
+ ids.add(id);
const config: Record = {};
for (const [key, item] of Object.entries(raw.config))
if (typeof item === "string") config[key] = item;
else throw new Error(`Stage ${index + 1} config values must be strings.`);
return {
- id: typeof raw.id === "string" ? raw.id : `stage-${index + 1}`,
+ id,
type: raw.type as StageType,
enabled: raw.enabled !== false,
config,
};
});
}
+function boundedInteger(
+ value: string,
+ label: string,
+ minimum: number,
+ maximum: number,
+) {
+ const parsed = Number(value);
+ if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum)
+ throw new Error(
+ `${label} must be an integer from ${minimum} to ${maximum}.`,
+ );
+ return parsed;
+}
+
+function analyze(
+ input: Record[],
+ output: Record[],
+ snapshots: Snapshot[],
+): PipelineAnalysis {
+ const fields = new Set(output.flatMap(Object.keys));
+ let nullCells = 0;
+ for (const row of output)
+ for (const field of fields) if (row[field] == null) nullCells += 1;
+ const identities = input.map((row) => stableStringify(row));
+ return {
+ inputRows: input.length,
+ outputRows: output.length,
+ fields: fields.size,
+ nullCells,
+ duplicateRows: identities.length - new Set(identities).size,
+ stageRows: snapshots.map((snapshot) => snapshot.rows.length),
+ };
+}
+
+export function renderPipelineSvg(stages: readonly Stage[]): string {
+ const enabled = stages.filter((stage) => stage.enabled);
+ const width = Math.max(520, Math.min(2_400, 160 + enabled.length * 190));
+ const nodes = ["Input", ...enabled.map((stage) => stage.type), "Result"];
+ const body = nodes
+ .map((label, index) => {
+ const x = 20 + index * ((width - 160) / Math.max(1, nodes.length - 1));
+ const box = ``;
+ const text = `${escapeXml(label)}`;
+ const previousX =
+ 20 + (index - 1) * ((width - 160) / Math.max(1, nodes.length - 1));
+ const edge = index
+ ? ``
+ : "";
+ return edge + box + text;
+ })
+ .join("");
+ return ``;
+}
+
+function escapeXml(value: string) {
+ return value
+ .replaceAll("&", "&")
+ .replaceAll("<", "<")
+ .replaceAll(">", ">")
+ .replaceAll('"', """);
+}
function list(value = "") {
return value
.split(",")
diff --git a/src/styles.css b/src/styles.css
index f3319bf..d115a06 100644
--- a/src/styles.css
+++ b/src/styles.css
@@ -82,6 +82,52 @@ button {
button:hover {
border-color: var(--accent);
}
+.file-button {
+ display: inline-flex;
+ align-items: center;
+ border: 1px solid var(--line);
+ border-radius: 0.6rem;
+ padding: 0.55rem 0.7rem;
+ color: CanvasText;
+ cursor: pointer;
+}
+.file-button input {
+ position: absolute;
+ width: 1px;
+ height: 1px;
+ overflow: hidden;
+ clip: rect(0 0 0 0);
+}
+.analysis-grid {
+ display: grid;
+ grid-template-columns: repeat(4, minmax(0, 1fr));
+ gap: 0.55rem;
+ margin: 0.7rem 0;
+}
+.analysis-grid div {
+ padding: 0.55rem;
+ border: 1px solid var(--line);
+ border-radius: 0.55rem;
+}
+.analysis-grid :where(dt, dd) {
+ margin: 0;
+}
+.analysis-grid dt {
+ color: var(--muted);
+ font-size: 0.75rem;
+}
+.analysis-grid dd {
+ font-weight: 700;
+}
+.run-actions {
+ display: flex;
+ gap: 0.5rem;
+ flex-wrap: wrap;
+}
+progress {
+ width: 100%;
+ accent-color: var(--accent);
+}
label {
display: grid;
gap: 0.25rem;
@@ -199,3 +245,8 @@ dialog {
flex-direction: column;
}
}
+@media (max-width: 540px) {
+ .analysis-grid {
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ }
+}
diff --git a/src/toolbox/manifest.source.json b/src/toolbox/manifest.source.json
index 6e7178f..9015f88 100644
--- a/src/toolbox/manifest.source.json
+++ b/src/toolbox/manifest.source.json
@@ -3,7 +3,7 @@
"schemaVersion": 1,
"id": "de.add-ideas.flow-tools",
"name": "Flow Tools",
- "version": "0.1.0",
+ "version": "0.2.0",
"description": "Transform structured data locally.",
"entry": "./",
"icon": "./favicon.svg",
@@ -16,11 +16,49 @@
},
"requirements": {
"secureContext": false,
- "workers": false,
+ "workers": true,
"indexedDb": false,
"crossOriginIsolated": false,
"topLevelContext": false
},
+ "io": {
+ "accepts": [
+ {
+ "mediaType": "application/json",
+ "extensions": [".json"]
+ },
+ {
+ "mediaType": "application/x-ndjson",
+ "extensions": [".ndjson", ".jsonl"]
+ },
+ {
+ "mediaType": "text/csv",
+ "extensions": [".csv"]
+ },
+ {
+ "mediaType": "application/xml",
+ "extensions": [".xml"]
+ }
+ ],
+ "produces": [
+ {
+ "mediaType": "application/json",
+ "extensions": [".json"]
+ },
+ {
+ "mediaType": "text/csv",
+ "extensions": [".csv"]
+ },
+ {
+ "mediaType": "image/svg+xml",
+ "extensions": [".svg"]
+ }
+ ]
+ },
+ "capabilities": {
+ "required": ["workers"],
+ "optional": []
+ },
"privacy": {
"processing": "local",
"fileUploads": true,
diff --git a/src/version.ts b/src/version.ts
index 76162f8..0ba6e37 100644
--- a/src/version.ts
+++ b/src/version.ts
@@ -1 +1 @@
-export const APP_VERSION = "0.1.0";
+export const APP_VERSION = "0.2.0";
diff --git a/tests/browser/app.spec.ts b/tests/browser/app.spec.ts
index cebad7f..acbcc7d 100644
--- a/tests/browser/app.spec.ts
+++ b/tests/browser/app.spec.ts
@@ -58,6 +58,6 @@ test("PWA theme, offline reload, and manifest", async ({
const m = await request.get("/deep/nested/flow/toolbox-app.json");
await expect(m.json()).resolves.toMatchObject({
id: "de.add-ideas.flow-tools",
- version: "0.1.0",
+ version: "0.2.0",
});
});
diff --git a/tests/browser/responsive.spec.ts b/tests/browser/responsive.spec.ts
new file mode 100644
index 0000000..5020122
--- /dev/null
+++ b/tests/browser/responsive.spec.ts
@@ -0,0 +1,18 @@
+import { expect, test } from "@playwright/test";
+
+test("keeps the primary workspace inside a narrow viewport", async ({
+ page,
+}) => {
+ await page.goto("/deep/nested/flow/");
+ await expect(page.locator("main").first()).toBeVisible();
+ await expect(
+ page.locator("main .loading, main .workbench-loading"),
+ ).toHaveCount(0);
+
+ const widths = await page.evaluate(() => ({
+ content: document.documentElement.scrollWidth,
+ viewport: document.documentElement.clientWidth,
+ }));
+ expect(widths.viewport).toBeLessThanOrEqual(430);
+ expect(widths.content).toBeLessThanOrEqual(widths.viewport + 1);
+});
diff --git a/tests/core/pipeline.test.ts b/tests/core/pipeline.test.ts
index 9385b8f..4c5ec11 100644
--- a/tests/core/pipeline.test.ts
+++ b/tests/core/pipeline.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
importRecipe,
records,
+ renderPipelineSvg,
runPipeline,
type Stage,
} from "../../src/core/pipeline";
@@ -55,5 +56,55 @@ describe("pipeline", () => {
'{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"type":"javascript","config":{}}]}',
),
).toThrow(/Invalid recipe/iu);
+ expect(() =>
+ importRecipe(
+ '{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"id":"same","type":"sort","enabled":true,"config":{}},{"id":"same","type":"sort","enabled":true,"config":{}}]}',
+ ),
+ ).toThrow(/safe unique id/iu);
+ });
+ it("deduplicates, explodes arrays and slices deterministically", () => {
+ const result = runPipeline(
+ records([
+ { id: 1, tags: ["a", "b"] },
+ { id: 1, tags: ["a", "b"] },
+ { id: 2, tags: ["c"] },
+ ]),
+ [
+ {
+ id: "d",
+ type: "deduplicate",
+ enabled: true,
+ config: { fields: "id" },
+ },
+ {
+ id: "e",
+ type: "explode",
+ enabled: true,
+ config: { field: "tags", target: "tag" },
+ },
+ {
+ id: "s",
+ type: "slice",
+ enabled: true,
+ config: { offset: "1", count: "2" },
+ },
+ ],
+ );
+ expect(result.rows.map((row) => row.tag)).toEqual(["b", "c"]);
+ expect(result.analysis).toMatchObject({
+ inputRows: 3,
+ outputRows: 2,
+ duplicateRows: 1,
+ });
+ expect(result.snapshots.at(-1)?.losses).toContain(
+ "1 rows fall outside the selected slice.",
+ );
+ });
+ it("exports an inert pipeline SVG", () => {
+ const svg = renderPipelineSvg([
+ { id: "x", type: "map", enabled: true, config: {} },
+ ]);
+ expect(svg).toContain("Flow pipeline");
+ expect(svg).not.toContain("