From 57dd710275a7c8bf26b4e42780db9179ee945dfc Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Wed, 2 Sep 2026 08:10:52 +0200 Subject: [PATCH] Release Flow Tools 0.2.0 --- .gitea/workflows/verify.yml | 39 ++++ CHANGELOG.md | 6 + README.md | 13 +- SOURCE.md | 2 +- THIRD_PARTY_NOTICES.md | 4 +- docs/ARCHITECTURE.md | 11 +- package-lock.json | 40 ++-- package.json | 13 +- playwright.config.ts | 22 ++- public/CHANGELOG.md | 6 + public/LICENSES/npm-runtime-licenses.txt | 6 +- public/README.md | 13 +- public/SOURCE.md | 2 +- public/THIRD_PARTY_NOTICES.md | 4 +- public/docs/ARCHITECTURE.md | 11 +- public/sw.js | 2 +- public/toolbox-app.json | 21 +- src/components/HelpDialog.tsx | 7 +- src/components/Workbench.tsx | 232 +++++++++++++++++++++-- src/core/flow-client.ts | 57 ++++++ src/core/flow.worker.ts | 31 +++ src/core/pipeline.ts | 168 +++++++++++++++- src/styles.css | 51 +++++ src/toolbox/manifest.source.json | 42 +++- src/version.ts | 2 +- tests/browser/app.spec.ts | 2 +- tests/browser/responsive.spec.ts | 18 ++ tests/core/pipeline.test.ts | 51 +++++ 28 files changed, 810 insertions(+), 66 deletions(-) create mode 100644 .gitea/workflows/verify.yml create mode 100644 src/core/flow-client.ts create mode 100644 src/core/flow.worker.ts create mode 100644 tests/browser/responsive.spec.ts diff --git a/.gitea/workflows/verify.yml b/.gitea/workflows/verify.yml new file mode 100644 index 0000000..b84a7cb --- /dev/null +++ b/.gitea/workflows/verify.yml @@ -0,0 +1,39 @@ +name: Verify + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: verify-${{ gitea.repository }}-${{ gitea.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + CI: "true" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + - name: Select declared npm version + run: npm install --global npm@11.17.0 + - name: Install dependencies + run: npm ci + - name: Audit runtime dependencies + run: npm audit --omit=dev --audit-level=moderate + - name: Check, test, and build + run: npm run check + - name: Install browser engines + run: npx playwright install --with-deps chromium firefox webkit + - name: Browser tests + run: npm run test:browser diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c2ba54..b3b995a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.2.0 - 2026-09-02 + +- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue, + explicit schema/loss evidence, cancellable worker execution and reusable + recipe/export handoffs. + ## 0.1.0 - 2026-09-01 - Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export. diff --git a/README.md b/README.md index 8b3d5dc..8cee735 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,14 @@ # Flow Tools -Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage. +Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted +or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, +sort, group, join, type-format, deduplicate, slice and explode primitives. Every +stage has an inspectable snapshot, dataset analysis and explicit loss notes. +Recipes are deterministic validated JSON; there is no arbitrary code, eval, +plugin or network stage. -Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`. +Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30 +stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result +JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`, +`npm run check`, `npm run test:browser`, then +`npm run package:release -- --force`. diff --git a/SOURCE.md b/SOURCE.md index c66081b..ebee4a3 100644 --- a/SOURCE.md +++ b/SOURCE.md @@ -1,3 +1,3 @@ # Source identity -Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar. +Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index b03e91e..4de531c 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,3 +1,5 @@ # Third-party notices -Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets. +Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox +Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0 +(GPL-3.0-or-later). Exact texts are generated in release assets. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 69124d3..8abd343 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,3 +1,12 @@ # Architecture -The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run. +The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and +produces JSON values. The interpreter copies rows stage-by-stage and accepts a +fixed discriminated grammar, including deduplication, bounded slicing and array +explosion. It records output, loss notes and dataset statistics after each run. + +Analysis and simulation run in a disposable module worker. Progress messages +are revision-checked, cancellation terminates the worker, and React retains the +last successful snapshots. Recipe imports are size-bounded and validate every +field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline +SVG are generated entirely in the browser. diff --git a/package-lock.json b/package-lock.json index 7dc5db1..2243f52 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,23 +1,23 @@ { "name": "flow-tools", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "flow-tools", - "version": "0.1.0", + "version": "0.2.0", "license": "GPL-3.0-or-later", "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3", - "@add-ideas/toolbox-helpers": "0.1.0", - "@add-ideas/toolbox-shell-react": "0.2.3", + "@add-ideas/toolbox-contract": "0.3.0", + "@add-ideas/toolbox-helpers": "0.2.0", + "@add-ideas/toolbox-shell-react": "0.3.0", "@xmldom/xmldom": "0.9.12", "react": "19.2.8", "react-dom": "19.2.8" }, "devDependencies": { - "@add-ideas/toolbox-testkit": "0.2.3", + "@add-ideas/toolbox-testkit": "0.3.0", "@eslint/js": "10.0.1", "@playwright/test": "1.62.1", "@testing-library/jest-dom": "6.9.1", @@ -43,24 +43,24 @@ } }, "node_modules/@add-ideas/toolbox-contract": { - "version": "0.2.3", - "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.2.3/toolbox-contract-0.2.3.tgz", - "integrity": "sha512-T0PVSuMT40GjTDfQJhEEY3ZawQq8zz1/ry95JdKI6W39CdLacaRXdGnEpDCMHt+jUbf1Jz7Nat/M5dFCgKVM9A==", + "version": "0.3.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz", + "integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==", "license": "Apache-2.0" }, "node_modules/@add-ideas/toolbox-helpers": { - "version": "0.1.0", - "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.1.0/toolbox-helpers-0.1.0.tgz", - "integrity": "sha512-UKl1Oxekedf8D2df86VrnVA53AcMhrnh6iUPXY+k8frirBXotb0yd8SGT+IF/3hcqYwcYe/v9WVFuSgKtIYVnw==", + "version": "0.2.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.2.0/toolbox-helpers-0.2.0.tgz", + "integrity": "sha512-SdOqkw+P+3J3fa5iVkzb5P15rVepB001GNV21Oh8w0CZcVL+YRltgD/s+MVcTyrNijWQf3E5vtQON/3N2LLyKg==", "license": "GPL-3.0-or-later" }, "node_modules/@add-ideas/toolbox-shell-react": { - "version": "0.2.3", - "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.2.3/toolbox-shell-react-0.2.3.tgz", - "integrity": "sha512-DT5lQDH48BFkFcmFLZnQh7+Cm73JzBPcmp5WzUXypfkUXpEyDYHzaXgmW4kZ0edSwh4RK4sPmx+JPtK0X4aKCQ==", + "version": "0.3.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz", + "integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==", "license": "Apache-2.0", "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3" + "@add-ideas/toolbox-contract": "0.3.0" }, "peerDependencies": { "react": ">=18 <20", @@ -68,13 +68,13 @@ } }, "node_modules/@add-ideas/toolbox-testkit": { - "version": "0.2.3", - "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.2.3/toolbox-testkit-0.2.3.tgz", - "integrity": "sha512-sq1MwhKWfFKen+N+124hl74qQimRSvmQ9sOU7jdcI+2qCKZ67+2B8rWyezeV80uTFu4Jv6deHksfYQ/tKNV6XQ==", + "version": "0.3.0", + "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz", + "integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3" + "@add-ideas/toolbox-contract": "0.3.0" }, "bin": { "toolbox-check": "dist/cli.js" diff --git a/package.json b/package.json index 3127f09..c9b4507 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flow-tools", - "version": "0.1.0", + "version": "0.2.0", "description": "Build and inspect bounded local data transformation pipelines.", "license": "GPL-3.0-or-later", "author": "Albrecht Degering", @@ -9,6 +9,9 @@ "url": "git+https://git.add-ideas.de/lotobo/flow-tools.git" }, "homepage": "https://git.add-ideas.de/lotobo/flow-tools", + "bugs": { + "url": "https://git.add-ideas.de/lotobo/flow-tools/issues" + }, "private": true, "type": "module", "engines": { @@ -34,15 +37,15 @@ "release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force" }, "dependencies": { - "@add-ideas/toolbox-contract": "0.2.3", - "@add-ideas/toolbox-helpers": "0.1.0", - "@add-ideas/toolbox-shell-react": "0.2.3", + "@add-ideas/toolbox-contract": "0.3.0", + "@add-ideas/toolbox-helpers": "0.2.0", + "@add-ideas/toolbox-shell-react": "0.3.0", "@xmldom/xmldom": "0.9.12", "react": "19.2.8", "react-dom": "19.2.8" }, "devDependencies": { - "@add-ideas/toolbox-testkit": "0.2.3", + "@add-ideas/toolbox-testkit": "0.3.0", "@eslint/js": "10.0.1", "@playwright/test": "1.62.1", "@testing-library/jest-dom": "6.9.1", diff --git a/playwright.config.ts b/playwright.config.ts index 74bc44f..af8741f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -12,7 +12,25 @@ export default defineConfig({ timeout: 180000, }, projects: [ - { name: "chromium", use: { ...devices["Desktop Chrome"] } }, - { name: "firefox", use: { ...devices["Desktop Firefox"] } }, + { + name: "chromium", + testIgnore: /responsive\.spec\.ts/, + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "firefox", + testIgnore: /responsive\.spec\.ts/, + use: { ...devices["Desktop Firefox"] }, + }, + { + name: "webkit", + testIgnore: /responsive\.spec\.ts/, + use: { ...devices["Desktop Safari"] }, + }, + { + name: "mobile-chromium", + testMatch: /responsive\.spec\.ts/, + use: { ...devices["Pixel 5"] }, + }, ], }); diff --git a/public/CHANGELOG.md b/public/CHANGELOG.md index 9c2ba54..b3b995a 100644 --- a/public/CHANGELOG.md +++ b/public/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.2.0 - 2026-09-02 + +- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue, + explicit schema/loss evidence, cancellable worker execution and reusable + recipe/export handoffs. + ## 0.1.0 - 2026-09-01 - Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export. diff --git a/public/LICENSES/npm-runtime-licenses.txt b/public/LICENSES/npm-runtime-licenses.txt index 647441c..c1c48cd 100644 --- a/public/LICENSES/npm-runtime-licenses.txt +++ b/public/LICENSES/npm-runtime-licenses.txt @@ -1,5 +1,5 @@ ============================================================================== -@add-ideas/toolbox-contract@0.2.3 +@add-ideas/toolbox-contract@0.3.0 Declared licence: Apache-2.0 ============================================================================== --- LICENSE --- @@ -198,7 +198,7 @@ Declared licence: Apache-2.0 ============================================================================== -@add-ideas/toolbox-helpers@0.1.0 +@add-ideas/toolbox-helpers@0.2.0 Declared licence: GPL-3.0-or-later ============================================================================== --- LICENSE --- @@ -879,7 +879,7 @@ Public License instead of this License. But first, please read ============================================================================== -@add-ideas/toolbox-shell-react@0.2.3 +@add-ideas/toolbox-shell-react@0.3.0 Declared licence: Apache-2.0 ============================================================================== --- LICENSE --- diff --git a/public/README.md b/public/README.md index 8b3d5dc..8cee735 100644 --- a/public/README.md +++ b/public/README.md @@ -1,5 +1,14 @@ # Flow Tools -Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage. +Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted +or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, +sort, group, join, type-format, deduplicate, slice and explode primitives. Every +stage has an inspectable snapshot, dataset analysis and explicit loss notes. +Recipes are deterministic validated JSON; there is no arbitrary code, eval, +plugin or network stage. -Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`. +Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30 +stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result +JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`, +`npm run check`, `npm run test:browser`, then +`npm run package:release -- --force`. diff --git a/public/SOURCE.md b/public/SOURCE.md index c66081b..ebee4a3 100644 --- a/public/SOURCE.md +++ b/public/SOURCE.md @@ -1,3 +1,3 @@ # Source identity -Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar. +Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar. diff --git a/public/THIRD_PARTY_NOTICES.md b/public/THIRD_PARTY_NOTICES.md index b03e91e..4de531c 100644 --- a/public/THIRD_PARTY_NOTICES.md +++ b/public/THIRD_PARTY_NOTICES.md @@ -1,3 +1,5 @@ # Third-party notices -Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets. +Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox +Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0 +(GPL-3.0-or-later). Exact texts are generated in release assets. diff --git a/public/docs/ARCHITECTURE.md b/public/docs/ARCHITECTURE.md index 69124d3..8abd343 100644 --- a/public/docs/ARCHITECTURE.md +++ b/public/docs/ARCHITECTURE.md @@ -1,3 +1,12 @@ # Architecture -The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run. +The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and +produces JSON values. The interpreter copies rows stage-by-stage and accepts a +fixed discriminated grammar, including deduplication, bounded slicing and array +explosion. It records output, loss notes and dataset statistics after each run. + +Analysis and simulation run in a disposable module worker. Progress messages +are revision-checked, cancellation terminates the worker, and React retains the +last successful snapshots. Recipe imports are size-bounded and validate every +field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline +SVG are generated entirely in the browser. diff --git a/public/sw.js b/public/sw.js index 8463933..d4da2e2 100644 --- a/public/sw.js +++ b/public/sw.js @@ -1,4 +1,4 @@ -const CACHE = "flow-tools-v0.1.0", +const CACHE = "flow-tools-v0.2.0", APP = [ "./", "./index.html", diff --git a/public/toolbox-app.json b/public/toolbox-app.json index 6e7178f..9c8663a 100644 --- a/public/toolbox-app.json +++ b/public/toolbox-app.json @@ -3,7 +3,7 @@ "schemaVersion": 1, "id": "de.add-ideas.flow-tools", "name": "Flow Tools", - "version": "0.1.0", + "version": "0.2.0", "description": "Transform structured data locally.", "entry": "./", "icon": "./favicon.svg", @@ -16,11 +16,28 @@ }, "requirements": { "secureContext": false, - "workers": false, + "workers": true, "indexedDb": false, "crossOriginIsolated": false, "topLevelContext": false }, + "io": { + "accepts": [ + { "mediaType": "application/json", "extensions": [".json"] }, + { + "mediaType": "application/x-ndjson", + "extensions": [".ndjson", ".jsonl"] + }, + { "mediaType": "text/csv", "extensions": [".csv"] }, + { "mediaType": "application/xml", "extensions": [".xml"] } + ], + "produces": [ + { "mediaType": "application/json", "extensions": [".json"] }, + { "mediaType": "text/csv", "extensions": [".csv"] }, + { "mediaType": "image/svg+xml", "extensions": [".svg"] } + ] + }, + "capabilities": { "required": ["workers"], "optional": [] }, "privacy": { "processing": "local", "fileUploads": true, diff --git a/src/components/HelpDialog.tsx b/src/components/HelpDialog.tsx index 3518f9d..ac65c6e 100644 --- a/src/components/HelpDialog.tsx +++ b/src/components/HelpDialog.tsx @@ -18,7 +18,12 @@ export function HelpDialog({ onCancel={onClose} aria-label="About Flow Tools" > -

Flow Tools

diff --git a/src/components/Workbench.tsx b/src/components/Workbench.tsx index f1090bc..c278c3b 100644 --- a/src/components/Workbench.tsx +++ b/src/components/Workbench.tsx @@ -1,4 +1,4 @@ -import { useMemo, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { stableStringify, stringifyCsv, @@ -6,10 +6,12 @@ import { type JsonValue, } from "@add-ideas/toolbox-helpers"; import { parseData, type DataFormat } from "../core/data"; +import { runFlowJob } from "../core/flow-client"; import { exportRecipe, importRecipe, records, + renderPipelineSvg, runPipeline, type PipelineResult, type Stage, @@ -48,7 +50,12 @@ export function Workbench() { ), [selected, setSelected] = useState(2), [status, setStatus] = useState("Sample pipeline completed locally."), - [recipeText, setRecipeText] = useState(""); + [recipeText, setRecipeText] = useState(""), + [busy, setBusy] = useState(false), + [progress, setProgress] = useState(0); + const operation = useRef<{ revision: number; controller?: AbortController }>({ + revision: 0, + }); const snapshot = result.snapshots[Math.min(selected, result.snapshots.length - 1)]!; function parsePrimary() { @@ -71,16 +78,42 @@ export function Workbench() { setStatus(`${msg(e)} Last valid secondary input retained.`); } } - function run() { + async function run() { + operation.current.controller?.abort(); + const controller = new AbortController(), + revision = ++operation.current.revision; + operation.current.controller = controller; + setBusy(true); + setProgress(0); try { - const next = runPipeline(input, stages, right); + const next = await runFlowJob(input, stages, right, { + signal: controller.signal, + onProgress: (item) => { + if (operation.current.revision !== revision) return; + setProgress(item.total ? item.completed / item.total : 0); + setStatus(`Running ${item.stage} locally…`); + }, + }); + if (operation.current.revision !== revision) return; setResult(next); setSelected(next.snapshots.length - 1); setStatus( `Completed ${stages.filter((s) => s.enabled).length} stages; ${next.rows.length} rows remain.`, ); } catch (e) { + if (operation.current.revision !== revision) return; + if (e instanceof DOMException && e.name === "AbortError") { + setStatus( + "Pipeline run cancelled. Last successful snapshots remain visible.", + ); + return; + } setStatus(`${msg(e)} Last successful stage snapshots remain visible.`); + } finally { + if (operation.current.revision === revision) { + setBusy(false); + operation.current.controller = undefined; + } } } function update(index: number, stage: Stage) { @@ -90,19 +123,24 @@ export function Workbench() { setStages((s) => [ ...s, { - id: `stage-${Date.now().toString(36)}`, + id: nextStageId(s), type, enabled: true, config: defaults(type), }, ]); } - function download(kind: "json" | "csv" | "recipe") { + function download(kind: "json" | "csv" | "recipe" | "svg") { if (kind === "recipe") triggerBlobDownload( new Blob([exportRecipe(stages)], { type: "application/json" }), "flow-recipe.json", ); + else if (kind === "svg") + triggerBlobDownload( + new Blob([renderPipelineSvg(stages)], { type: "image/svg+xml" }), + "flow-pipeline.svg", + ); else if (kind === "json") triggerBlobDownload( new Blob([stableStringify(result.rows, 2)], { @@ -124,6 +162,41 @@ export function Workbench() { ); } } + async function openData(file: File | undefined, secondary = false) { + if (!file) return; + try { + if (file.size > 2 * 1024 * 1024) + throw new Error("Input file exceeds the 2 MiB limit."); + const text = await file.text(), + detected = inferFormat(file.name), + next = records(parseData(text, detected).rows); + if (secondary) { + setRightSource(text); + setRight(next); + setStatus(`Opened ${next.length} secondary rows from ${file.name}.`); + } else { + setFormat(detected); + setSource(text); + setInput(next); + setStatus(`Opened ${next.length} primary rows from ${file.name}.`); + } + } catch (e) { + setStatus(`${msg(e)} Last valid input retained.`); + } + } + async function openRecipe(file: File | undefined) { + if (!file) return; + try { + if (file.size > 256 * 1024) throw new Error("Recipe exceeds 256 KiB."); + const text = await file.text(), + next = importRecipe(text); + setRecipeText(text); + setStages(next); + setStatus(`Imported ${next.length} validated stages from ${file.name}.`); + } catch (e) { + setStatus(msg(e)); + } + } function loadRecipe() { try { const next = importRecipe(recipeText); @@ -144,11 +217,25 @@ export function Workbench() { explicit losses with a deterministic recipe.

- +
+ + {busy && ( + + )} +

{status}

+ {busy && ( + + )}

Primary source

@@ -169,7 +256,17 @@ export function Workbench() { onChange={(e) => setSource(e.target.value)} aria-label="Primary source" /> - +
+ + +

Secondary join source

@@ -179,7 +276,19 @@ export function Workbench() { onChange={(e) => setRightSource(e.target.value)} aria-label="Secondary source" /> - +
+ + +
@@ -196,6 +305,9 @@ export function Workbench() { + + +