1 Commits
Author SHA1 Message Date
zemion 57dd710275 Release Flow Tools 0.2.0
Verify / verify (push) Canceled after 0s
2026-09-02 08:10:52 +02:00
28 changed files with 810 additions and 66 deletions
+39
View File
@@ -0,0 +1,39 @@
name: Verify
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
concurrency:
group: verify-${{ gitea.repository }}-${{ gitea.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
CI: "true"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Select declared npm version
run: npm install --global npm@11.17.0
- name: Install dependencies
run: npm ci
- name: Audit runtime dependencies
run: npm audit --omit=dev --audit-level=moderate
- name: Check, test, and build
run: npm run check
- name: Install browser engines
run: npx playwright install --with-deps chromium firefox webkit
- name: Browser tests
run: npm run test:browser
+6
View File
@@ -1,5 +1,11 @@
# Changelog # Changelog
## 0.2.0 - 2026-09-02
- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue,
explicit schema/loss evidence, cancellable worker execution and reusable
recipe/export handoffs.
## 0.1.0 - 2026-09-01 ## 0.1.0 - 2026-09-01
- Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export. - Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export.
+11 -2
View File
@@ -1,5 +1,14 @@
# Flow Tools # Flow Tools
Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage. Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted
or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map,
sort, group, join, type-format, deduplicate, slice and explode primitives. Every
stage has an inspectable snapshot, dataset analysis and explicit loss notes.
Recipes are deterministic validated JSON; there is no arbitrary code, eval,
plugin or network stage.
Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`. Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30
stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result
JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`,
`npm run check`, `npm run test:browser`, then
`npm run package:release -- --force`.
+1 -1
View File
@@ -1,3 +1,3 @@
# Source identity # Source identity
Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar. Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
+3 -1
View File
@@ -1,3 +1,5 @@
# Third-party notices # Third-party notices
Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets. Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox
Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0
(GPL-3.0-or-later). Exact texts are generated in release assets.
+10 -1
View File
@@ -1,3 +1,12 @@
# Architecture # Architecture
The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run. The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and
produces JSON values. The interpreter copies rows stage-by-stage and accepts a
fixed discriminated grammar, including deduplication, bounded slicing and array
explosion. It records output, loss notes and dataset statistics after each run.
Analysis and simulation run in a disposable module worker. Progress messages
are revision-checked, cancellation terminates the worker, and React retains the
last successful snapshots. Recipe imports are size-bounded and validate every
field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline
SVG are generated entirely in the browser.
+20 -20
View File
@@ -1,23 +1,23 @@
{ {
"name": "flow-tools", "name": "flow-tools",
"version": "0.1.0", "version": "0.2.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "flow-tools", "name": "flow-tools",
"version": "0.1.0", "version": "0.2.0",
"license": "GPL-3.0-or-later", "license": "GPL-3.0-or-later",
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3", "@add-ideas/toolbox-contract": "0.3.0",
"@add-ideas/toolbox-helpers": "0.1.0", "@add-ideas/toolbox-helpers": "0.2.0",
"@add-ideas/toolbox-shell-react": "0.2.3", "@add-ideas/toolbox-shell-react": "0.3.0",
"@xmldom/xmldom": "0.9.12", "@xmldom/xmldom": "0.9.12",
"react": "19.2.8", "react": "19.2.8",
"react-dom": "19.2.8" "react-dom": "19.2.8"
}, },
"devDependencies": { "devDependencies": {
"@add-ideas/toolbox-testkit": "0.2.3", "@add-ideas/toolbox-testkit": "0.3.0",
"@eslint/js": "10.0.1", "@eslint/js": "10.0.1",
"@playwright/test": "1.62.1", "@playwright/test": "1.62.1",
"@testing-library/jest-dom": "6.9.1", "@testing-library/jest-dom": "6.9.1",
@@ -43,24 +43,24 @@
} }
}, },
"node_modules/@add-ideas/toolbox-contract": { "node_modules/@add-ideas/toolbox-contract": {
"version": "0.2.3", "version": "0.3.0",
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.2.3/toolbox-contract-0.2.3.tgz", "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz",
"integrity": "sha512-T0PVSuMT40GjTDfQJhEEY3ZawQq8zz1/ry95JdKI6W39CdLacaRXdGnEpDCMHt+jUbf1Jz7Nat/M5dFCgKVM9A==", "integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==",
"license": "Apache-2.0" "license": "Apache-2.0"
}, },
"node_modules/@add-ideas/toolbox-helpers": { "node_modules/@add-ideas/toolbox-helpers": {
"version": "0.1.0", "version": "0.2.0",
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.1.0/toolbox-helpers-0.1.0.tgz", "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.2.0/toolbox-helpers-0.2.0.tgz",
"integrity": "sha512-UKl1Oxekedf8D2df86VrnVA53AcMhrnh6iUPXY+k8frirBXotb0yd8SGT+IF/3hcqYwcYe/v9WVFuSgKtIYVnw==", "integrity": "sha512-SdOqkw+P+3J3fa5iVkzb5P15rVepB001GNV21Oh8w0CZcVL+YRltgD/s+MVcTyrNijWQf3E5vtQON/3N2LLyKg==",
"license": "GPL-3.0-or-later" "license": "GPL-3.0-or-later"
}, },
"node_modules/@add-ideas/toolbox-shell-react": { "node_modules/@add-ideas/toolbox-shell-react": {
"version": "0.2.3", "version": "0.3.0",
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.2.3/toolbox-shell-react-0.2.3.tgz", "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz",
"integrity": "sha512-DT5lQDH48BFkFcmFLZnQh7+Cm73JzBPcmp5WzUXypfkUXpEyDYHzaXgmW4kZ0edSwh4RK4sPmx+JPtK0X4aKCQ==", "integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==",
"license": "Apache-2.0", "license": "Apache-2.0",
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3" "@add-ideas/toolbox-contract": "0.3.0"
}, },
"peerDependencies": { "peerDependencies": {
"react": ">=18 <20", "react": ">=18 <20",
@@ -68,13 +68,13 @@
} }
}, },
"node_modules/@add-ideas/toolbox-testkit": { "node_modules/@add-ideas/toolbox-testkit": {
"version": "0.2.3", "version": "0.3.0",
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.2.3/toolbox-testkit-0.2.3.tgz", "resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz",
"integrity": "sha512-sq1MwhKWfFKen+N+124hl74qQimRSvmQ9sOU7jdcI+2qCKZ67+2B8rWyezeV80uTFu4Jv6deHksfYQ/tKNV6XQ==", "integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==",
"dev": true, "dev": true,
"license": "Apache-2.0", "license": "Apache-2.0",
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3" "@add-ideas/toolbox-contract": "0.3.0"
}, },
"bin": { "bin": {
"toolbox-check": "dist/cli.js" "toolbox-check": "dist/cli.js"
+8 -5
View File
@@ -1,6 +1,6 @@
{ {
"name": "flow-tools", "name": "flow-tools",
"version": "0.1.0", "version": "0.2.0",
"description": "Build and inspect bounded local data transformation pipelines.", "description": "Build and inspect bounded local data transformation pipelines.",
"license": "GPL-3.0-or-later", "license": "GPL-3.0-or-later",
"author": "Albrecht Degering", "author": "Albrecht Degering",
@@ -9,6 +9,9 @@
"url": "git+https://git.add-ideas.de/lotobo/flow-tools.git" "url": "git+https://git.add-ideas.de/lotobo/flow-tools.git"
}, },
"homepage": "https://git.add-ideas.de/lotobo/flow-tools", "homepage": "https://git.add-ideas.de/lotobo/flow-tools",
"bugs": {
"url": "https://git.add-ideas.de/lotobo/flow-tools/issues"
},
"private": true, "private": true,
"type": "module", "type": "module",
"engines": { "engines": {
@@ -34,15 +37,15 @@
"release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force" "release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force"
}, },
"dependencies": { "dependencies": {
"@add-ideas/toolbox-contract": "0.2.3", "@add-ideas/toolbox-contract": "0.3.0",
"@add-ideas/toolbox-helpers": "0.1.0", "@add-ideas/toolbox-helpers": "0.2.0",
"@add-ideas/toolbox-shell-react": "0.2.3", "@add-ideas/toolbox-shell-react": "0.3.0",
"@xmldom/xmldom": "0.9.12", "@xmldom/xmldom": "0.9.12",
"react": "19.2.8", "react": "19.2.8",
"react-dom": "19.2.8" "react-dom": "19.2.8"
}, },
"devDependencies": { "devDependencies": {
"@add-ideas/toolbox-testkit": "0.2.3", "@add-ideas/toolbox-testkit": "0.3.0",
"@eslint/js": "10.0.1", "@eslint/js": "10.0.1",
"@playwright/test": "1.62.1", "@playwright/test": "1.62.1",
"@testing-library/jest-dom": "6.9.1", "@testing-library/jest-dom": "6.9.1",
+20 -2
View File
@@ -12,7 +12,25 @@ export default defineConfig({
timeout: 180000, timeout: 180000,
}, },
projects: [ projects: [
{ name: "chromium", use: { ...devices["Desktop Chrome"] } }, {
{ name: "firefox", use: { ...devices["Desktop Firefox"] } }, name: "chromium",
testIgnore: /responsive\.spec\.ts/,
use: { ...devices["Desktop Chrome"] },
},
{
name: "firefox",
testIgnore: /responsive\.spec\.ts/,
use: { ...devices["Desktop Firefox"] },
},
{
name: "webkit",
testIgnore: /responsive\.spec\.ts/,
use: { ...devices["Desktop Safari"] },
},
{
name: "mobile-chromium",
testMatch: /responsive\.spec\.ts/,
use: { ...devices["Pixel 5"] },
},
], ],
}); });
+6
View File
@@ -1,5 +1,11 @@
# Changelog # Changelog
## 0.2.0 - 2026-09-02
- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue,
explicit schema/loss evidence, cancellable worker execution and reusable
recipe/export handoffs.
## 0.1.0 - 2026-09-01 ## 0.1.0 - 2026-09-01
- Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export. - Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export.
+3 -3
View File
@@ -1,5 +1,5 @@
============================================================================== ==============================================================================
@add-ideas/toolbox-contract@0.2.3 @add-ideas/toolbox-contract@0.3.0
Declared licence: Apache-2.0 Declared licence: Apache-2.0
============================================================================== ==============================================================================
--- LICENSE --- --- LICENSE ---
@@ -198,7 +198,7 @@ Declared licence: Apache-2.0
============================================================================== ==============================================================================
@add-ideas/toolbox-helpers@0.1.0 @add-ideas/toolbox-helpers@0.2.0
Declared licence: GPL-3.0-or-later Declared licence: GPL-3.0-or-later
============================================================================== ==============================================================================
--- LICENSE --- --- LICENSE ---
@@ -879,7 +879,7 @@ Public License instead of this License. But first, please read
============================================================================== ==============================================================================
@add-ideas/toolbox-shell-react@0.2.3 @add-ideas/toolbox-shell-react@0.3.0
Declared licence: Apache-2.0 Declared licence: Apache-2.0
============================================================================== ==============================================================================
--- LICENSE --- --- LICENSE ---
+11 -2
View File
@@ -1,5 +1,14 @@
# Flow Tools # Flow Tools
Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage. Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted
or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map,
sort, group, join, type-format, deduplicate, slice and explode primitives. Every
stage has an inspectable snapshot, dataset analysis and explicit loss notes.
Recipes are deterministic validated JSON; there is no arbitrary code, eval,
plugin or network stage.
Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`. Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30
stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result
JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`,
`npm run check`, `npm run test:browser`, then
`npm run package:release -- --force`.
+1 -1
View File
@@ -1,3 +1,3 @@
# Source identity # Source identity
Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar. Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
+3 -1
View File
@@ -1,3 +1,5 @@
# Third-party notices # Third-party notices
Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets. Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox
Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0
(GPL-3.0-or-later). Exact texts are generated in release assets.
+10 -1
View File
@@ -1,3 +1,12 @@
# Architecture # Architecture
The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run. The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and
produces JSON values. The interpreter copies rows stage-by-stage and accepts a
fixed discriminated grammar, including deduplication, bounded slicing and array
explosion. It records output, loss notes and dataset statistics after each run.
Analysis and simulation run in a disposable module worker. Progress messages
are revision-checked, cancellation terminates the worker, and React retains the
last successful snapshots. Recipe imports are size-bounded and validate every
field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline
SVG are generated entirely in the browser.
+1 -1
View File
@@ -1,4 +1,4 @@
const CACHE = "flow-tools-v0.1.0", const CACHE = "flow-tools-v0.2.0",
APP = [ APP = [
"./", "./",
"./index.html", "./index.html",
+19 -2
View File
@@ -3,7 +3,7 @@
"schemaVersion": 1, "schemaVersion": 1,
"id": "de.add-ideas.flow-tools", "id": "de.add-ideas.flow-tools",
"name": "Flow Tools", "name": "Flow Tools",
"version": "0.1.0", "version": "0.2.0",
"description": "Transform structured data locally.", "description": "Transform structured data locally.",
"entry": "./", "entry": "./",
"icon": "./favicon.svg", "icon": "./favicon.svg",
@@ -16,11 +16,28 @@
}, },
"requirements": { "requirements": {
"secureContext": false, "secureContext": false,
"workers": false, "workers": true,
"indexedDb": false, "indexedDb": false,
"crossOriginIsolated": false, "crossOriginIsolated": false,
"topLevelContext": false "topLevelContext": false
}, },
"io": {
"accepts": [
{ "mediaType": "application/json", "extensions": [".json"] },
{
"mediaType": "application/x-ndjson",
"extensions": [".ndjson", ".jsonl"]
},
{ "mediaType": "text/csv", "extensions": [".csv"] },
{ "mediaType": "application/xml", "extensions": [".xml"] }
],
"produces": [
{ "mediaType": "application/json", "extensions": [".json"] },
{ "mediaType": "text/csv", "extensions": [".csv"] },
{ "mediaType": "image/svg+xml", "extensions": [".svg"] }
]
},
"capabilities": { "required": ["workers"], "optional": [] },
"privacy": { "privacy": {
"processing": "local", "processing": "local",
"fileUploads": true, "fileUploads": true,
+6 -1
View File
@@ -18,7 +18,12 @@ export function HelpDialog({
onCancel={onClose} onCancel={onClose}
aria-label="About Flow Tools" aria-label="About Flow Tools"
> >
<button className="close" onClick={onClose}> <button
type="button"
className="close"
aria-label="Close help"
onClick={onClose}
>
× ×
</button> </button>
<h2>Flow Tools</h2> <h2>Flow Tools</h2>
+214 -10
View File
@@ -1,4 +1,4 @@
import { useMemo, useState } from "react"; import { useMemo, useRef, useState } from "react";
import { import {
stableStringify, stableStringify,
stringifyCsv, stringifyCsv,
@@ -6,10 +6,12 @@ import {
type JsonValue, type JsonValue,
} from "@add-ideas/toolbox-helpers"; } from "@add-ideas/toolbox-helpers";
import { parseData, type DataFormat } from "../core/data"; import { parseData, type DataFormat } from "../core/data";
import { runFlowJob } from "../core/flow-client";
import { import {
exportRecipe, exportRecipe,
importRecipe, importRecipe,
records, records,
renderPipelineSvg,
runPipeline, runPipeline,
type PipelineResult, type PipelineResult,
type Stage, type Stage,
@@ -48,7 +50,12 @@ export function Workbench() {
), ),
[selected, setSelected] = useState(2), [selected, setSelected] = useState(2),
[status, setStatus] = useState("Sample pipeline completed locally."), [status, setStatus] = useState("Sample pipeline completed locally."),
[recipeText, setRecipeText] = useState(""); [recipeText, setRecipeText] = useState(""),
[busy, setBusy] = useState(false),
[progress, setProgress] = useState(0);
const operation = useRef<{ revision: number; controller?: AbortController }>({
revision: 0,
});
const snapshot = const snapshot =
result.snapshots[Math.min(selected, result.snapshots.length - 1)]!; result.snapshots[Math.min(selected, result.snapshots.length - 1)]!;
function parsePrimary() { function parsePrimary() {
@@ -71,16 +78,42 @@ export function Workbench() {
setStatus(`${msg(e)} Last valid secondary input retained.`); setStatus(`${msg(e)} Last valid secondary input retained.`);
} }
} }
function run() { async function run() {
operation.current.controller?.abort();
const controller = new AbortController(),
revision = ++operation.current.revision;
operation.current.controller = controller;
setBusy(true);
setProgress(0);
try { try {
const next = runPipeline(input, stages, right); const next = await runFlowJob(input, stages, right, {
signal: controller.signal,
onProgress: (item) => {
if (operation.current.revision !== revision) return;
setProgress(item.total ? item.completed / item.total : 0);
setStatus(`Running ${item.stage} locally…`);
},
});
if (operation.current.revision !== revision) return;
setResult(next); setResult(next);
setSelected(next.snapshots.length - 1); setSelected(next.snapshots.length - 1);
setStatus( setStatus(
`Completed ${stages.filter((s) => s.enabled).length} stages; ${next.rows.length} rows remain.`, `Completed ${stages.filter((s) => s.enabled).length} stages; ${next.rows.length} rows remain.`,
); );
} catch (e) { } catch (e) {
if (operation.current.revision !== revision) return;
if (e instanceof DOMException && e.name === "AbortError") {
setStatus(
"Pipeline run cancelled. Last successful snapshots remain visible.",
);
return;
}
setStatus(`${msg(e)} Last successful stage snapshots remain visible.`); setStatus(`${msg(e)} Last successful stage snapshots remain visible.`);
} finally {
if (operation.current.revision === revision) {
setBusy(false);
operation.current.controller = undefined;
}
} }
} }
function update(index: number, stage: Stage) { function update(index: number, stage: Stage) {
@@ -90,19 +123,24 @@ export function Workbench() {
setStages((s) => [ setStages((s) => [
...s, ...s,
{ {
id: `stage-${Date.now().toString(36)}`, id: nextStageId(s),
type, type,
enabled: true, enabled: true,
config: defaults(type), config: defaults(type),
}, },
]); ]);
} }
function download(kind: "json" | "csv" | "recipe") { function download(kind: "json" | "csv" | "recipe" | "svg") {
if (kind === "recipe") if (kind === "recipe")
triggerBlobDownload( triggerBlobDownload(
new Blob([exportRecipe(stages)], { type: "application/json" }), new Blob([exportRecipe(stages)], { type: "application/json" }),
"flow-recipe.json", "flow-recipe.json",
); );
else if (kind === "svg")
triggerBlobDownload(
new Blob([renderPipelineSvg(stages)], { type: "image/svg+xml" }),
"flow-pipeline.svg",
);
else if (kind === "json") else if (kind === "json")
triggerBlobDownload( triggerBlobDownload(
new Blob([stableStringify(result.rows, 2)], { new Blob([stableStringify(result.rows, 2)], {
@@ -124,6 +162,41 @@ export function Workbench() {
); );
} }
} }
async function openData(file: File | undefined, secondary = false) {
if (!file) return;
try {
if (file.size > 2 * 1024 * 1024)
throw new Error("Input file exceeds the 2 MiB limit.");
const text = await file.text(),
detected = inferFormat(file.name),
next = records(parseData(text, detected).rows);
if (secondary) {
setRightSource(text);
setRight(next);
setStatus(`Opened ${next.length} secondary rows from ${file.name}.`);
} else {
setFormat(detected);
setSource(text);
setInput(next);
setStatus(`Opened ${next.length} primary rows from ${file.name}.`);
}
} catch (e) {
setStatus(`${msg(e)} Last valid input retained.`);
}
}
async function openRecipe(file: File | undefined) {
if (!file) return;
try {
if (file.size > 256 * 1024) throw new Error("Recipe exceeds 256 KiB.");
const text = await file.text(),
next = importRecipe(text);
setRecipeText(text);
setStages(next);
setStatus(`Imported ${next.length} validated stages from ${file.name}.`);
} catch (e) {
setStatus(msg(e));
}
}
function loadRecipe() { function loadRecipe() {
try { try {
const next = importRecipe(recipeText); const next = importRecipe(recipeText);
@@ -144,11 +217,25 @@ export function Workbench() {
explicit losses with a deterministic recipe. explicit losses with a deterministic recipe.
</p> </p>
</div> </div>
<button className="primary" onClick={run}> <div className="run-actions">
<button
className="primary"
onClick={() => void run()}
disabled={busy}
>
Run complete pipeline Run complete pipeline
</button> </button>
{busy && (
<button onClick={() => operation.current.controller?.abort()}>
Cancel
</button>
)}
</div>
</section> </section>
<p role="status">{status}</p> <p role="status">{status}</p>
{busy && (
<progress value={progress} max={1} aria-label="Pipeline progress" />
)}
<section className="sources"> <section className="sources">
<section className="panel"> <section className="panel">
<h2>Primary source</h2> <h2>Primary source</h2>
@@ -169,7 +256,17 @@ export function Workbench() {
onChange={(e) => setSource(e.target.value)} onChange={(e) => setSource(e.target.value)}
aria-label="Primary source" aria-label="Primary source"
/> />
<div className="exports">
<button onClick={parsePrimary}>Parse primary</button> <button onClick={parsePrimary}>Parse primary</button>
<label className="file-button">
Open data file
<input
type="file"
accept=".json,.csv,.ndjson,.jsonl,.xml"
onChange={(event) => void openData(event.target.files?.[0])}
/>
</label>
</div>
</section> </section>
<section className="panel"> <section className="panel">
<h2>Secondary join source</h2> <h2>Secondary join source</h2>
@@ -179,7 +276,19 @@ export function Workbench() {
onChange={(e) => setRightSource(e.target.value)} onChange={(e) => setRightSource(e.target.value)}
aria-label="Secondary source" aria-label="Secondary source"
/> />
<div className="exports">
<button onClick={parseRight}>Parse secondary</button> <button onClick={parseRight}>Parse secondary</button>
<label className="file-button">
Open join file
<input
type="file"
accept=".json,.csv,.ndjson,.jsonl,.xml"
onChange={(event) =>
void openData(event.target.files?.[0], true)
}
/>
</label>
</div>
</section> </section>
</section> </section>
<section className="pipeline"> <section className="pipeline">
@@ -196,6 +305,9 @@ export function Workbench() {
<option value="group">Group</option> <option value="group">Group</option>
<option value="join">Join</option> <option value="join">Join</option>
<option value="format">Format type</option> <option value="format">Format type</option>
<option value="deduplicate">Remove duplicates</option>
<option value="slice">Slice rows</option>
<option value="explode">Explode array</option>
</select> </select>
</label> </label>
<button <button
@@ -250,10 +362,31 @@ export function Workbench() {
</ul> </ul>
)} )}
<Table rows={snapshot.rows} /> <Table rows={snapshot.rows} />
<dl className="analysis-grid" aria-label="Result analysis">
<div>
<dt>Rows</dt>
<dd>
{result.analysis.inputRows} {result.analysis.outputRows}
</dd>
</div>
<div>
<dt>Fields</dt>
<dd>{result.analysis.fields}</dd>
</div>
<div>
<dt>Empty cells</dt>
<dd>{result.analysis.nullCells}</dd>
</div>
<div>
<dt>Input duplicates</dt>
<dd>{result.analysis.duplicateRows}</dd>
</div>
</dl>
<div className="exports"> <div className="exports">
<button onClick={() => download("json")}>Result JSON</button> <button onClick={() => download("json")}>Result JSON</button>
<button onClick={() => download("csv")}>Result CSV</button> <button onClick={() => download("csv")}>Result CSV</button>
<button onClick={() => download("recipe")}>Recipe JSON</button> <button onClick={() => download("recipe")}>Recipe JSON</button>
<button onClick={() => download("svg")}>Pipeline SVG</button>
</div> </div>
<details> <details>
<summary>Import deterministic recipe</summary> <summary>Import deterministic recipe</summary>
@@ -263,6 +396,14 @@ export function Workbench() {
aria-label="Recipe JSON" aria-label="Recipe JSON"
/> />
<button onClick={loadRecipe}>Validate and import</button> <button onClick={loadRecipe}>Validate and import</button>
<label className="file-button">
Open recipe file
<input
type="file"
accept=".json,application/json"
onChange={(event) => void openRecipe(event.target.files?.[0])}
/>
</label>
</details> </details>
</section> </section>
</section> </section>
@@ -303,8 +444,20 @@ function StageEditor({
</header> </header>
<Config stage={stage} set={set} /> <Config stage={stage} set={set} />
<footer> <footer>
<button onClick={() => move(-1)}></button> <button
<button onClick={() => move(1)}></button> type="button"
aria-label="Move stage up"
onClick={() => move(-1)}
>
</button>
<button
type="button"
aria-label="Move stage down"
onClick={() => move(1)}
>
</button>
<button onClick={remove}>Remove</button> <button onClick={remove}>Remove</button>
</footer> </footer>
</article> </article>
@@ -415,6 +568,36 @@ function Config({
/> />
</> </>
); );
if (stage.type === "deduplicate")
return (
<Field
label="Identity fields (blank = whole row)"
value={f.fields}
set={(v) => set("fields", v)}
/>
);
if (stage.type === "slice")
return (
<>
<Field label="Offset" value={f.offset} set={(v) => set("offset", v)} />
<Field label="Count" value={f.count} set={(v) => set("count", v)} />
</>
);
if (stage.type === "explode")
return (
<>
<Field
label="Array field"
value={f.field}
set={(v) => set("field", v)}
/>
<Field
label="Output field"
value={f.target}
set={(v) => set("target", v)}
/>
</>
);
return ( return (
<> <>
<Field label="Target" value={f.target} set={(v) => set("target", v)} /> <Field label="Target" value={f.target} set={(v) => set("target", v)} />
@@ -530,5 +713,26 @@ function defaults(t: StageType): Record<string, string> {
mode: "left", mode: "left",
prefix: "right_", prefix: "right_",
} }
: { field: "score", as: "number" }; : t === "format"
? { field: "score", as: "number" }
: t === "deduplicate"
? { fields: "" }
: t === "slice"
? { offset: "0", count: "100" }
: { field: "items", target: "item" };
}
function inferFormat(fileName: string): DataFormat {
const extension = fileName.split(".").at(-1)?.toLowerCase();
if (extension === "csv") return "csv";
if (extension === "ndjson" || extension === "jsonl") return "ndjson";
if (extension === "xml") return "xml";
return "json";
}
function nextStageId(stages: readonly Stage[]): string {
const ids = new Set(stages.map((stage) => stage.id));
let suffix = 1;
while (ids.has(`stage-${suffix}`)) suffix += 1;
return `stage-${suffix}`;
} }
+57
View File
@@ -0,0 +1,57 @@
import type { JsonValue } from "@add-ideas/toolbox-helpers";
import {
runPipeline,
type PipelineProgress,
type PipelineResult,
type Stage,
} from "./pipeline";
export async function runFlowJob(
input: Record<string, JsonValue>[],
stages: Stage[],
right: Record<string, JsonValue>[],
options: {
signal?: AbortSignal;
onProgress?: (progress: PipelineProgress) => void;
} = {},
): Promise<PipelineResult> {
if (options.signal?.aborted) throw abortError();
if (typeof Worker === "undefined")
return runPipeline(input, stages, right, options.onProgress);
const worker = new Worker(new URL("./flow.worker.ts", import.meta.url), {
type: "module",
name: "flow-tools-analysis",
});
const id = Date.now() + Math.random();
return await new Promise<PipelineResult>((resolve, reject) => {
const cleanup = () => {
options.signal?.removeEventListener("abort", cancel);
worker.terminate();
};
const cancel = () => {
cleanup();
reject(abortError());
};
options.signal?.addEventListener("abort", cancel, { once: true });
worker.addEventListener("message", (event) => {
if (event.data?.id !== id) return;
if (event.data.type === "progress") {
options.onProgress?.(event.data.progress as PipelineProgress);
return;
}
cleanup();
if (event.data.type === "result")
resolve(event.data.result as PipelineResult);
else reject(new Error(String(event.data.error ?? "Flow worker failed.")));
});
worker.addEventListener("error", () => {
cleanup();
reject(new Error("The isolated flow worker stopped unexpectedly."));
});
worker.postMessage({ id, input, stages, right });
});
}
function abortError() {
return new DOMException("Flow operation cancelled.", "AbortError");
}
+31
View File
@@ -0,0 +1,31 @@
/// <reference lib="webworker" />
import { runPipeline, type PipelineProgress, type Stage } from "./pipeline";
import type { JsonValue } from "@add-ideas/toolbox-helpers";
interface Request {
id: number;
input: Record<string, JsonValue>[];
right: Record<string, JsonValue>[];
stages: Stage[];
}
const scope = self as DedicatedWorkerGlobalScope;
scope.addEventListener("message", (event: MessageEvent<Request>) => {
const request = event.data;
try {
const result = runPipeline(
request.input,
request.stages,
request.right,
(progress: PipelineProgress) =>
scope.postMessage({ id: request.id, type: "progress", progress }),
);
scope.postMessage({ id: request.id, type: "result", result });
} catch (error) {
scope.postMessage({
id: request.id,
type: "error",
error: error instanceof Error ? error.message : String(error),
});
}
});
+164 -4
View File
@@ -5,7 +5,17 @@ import {
} from "@add-ideas/toolbox-helpers"; } from "@add-ideas/toolbox-helpers";
import { isObject } from "./data"; import { isObject } from "./data";
export type StageType = export type StageType =
"select" | "rename" | "filter" | "map" | "sort" | "group" | "join" | "format"; | "select"
| "rename"
| "filter"
| "map"
| "sort"
| "group"
| "join"
| "format"
| "deduplicate"
| "slice"
| "explode";
export interface Stage { export interface Stage {
id: string; id: string;
type: StageType; type: StageType;
@@ -22,6 +32,20 @@ export interface PipelineResult {
snapshots: Snapshot[]; snapshots: Snapshot[];
rows: Record<string, JsonValue>[]; rows: Record<string, JsonValue>[];
recipe: string; recipe: string;
analysis: PipelineAnalysis;
}
export interface PipelineAnalysis {
inputRows: number;
outputRows: number;
fields: number;
nullCells: number;
duplicateRows: number;
stageRows: number[];
}
export interface PipelineProgress {
stage: string;
completed: number;
total: number;
} }
const MAX_STAGES = 30, const MAX_STAGES = 30,
MAX_ROWS = 10000, MAX_ROWS = 10000,
@@ -36,11 +60,13 @@ export function runPipeline(
input: Record<string, JsonValue>[], input: Record<string, JsonValue>[],
stages: Stage[], stages: Stage[],
right: Record<string, JsonValue>[] = [], right: Record<string, JsonValue>[] = [],
onProgress?: (progress: PipelineProgress) => void,
): PipelineResult { ): PipelineResult {
if (stages.length > MAX_STAGES) throw new Error("Recipe exceeds 30 stages."); if (stages.length > MAX_STAGES) throw new Error("Recipe exceeds 30 stages.");
let rows = input.map((row) => ({ ...row })); let rows = input.map((row) => ({ ...row }));
const snapshots: Snapshot[] = [{ name: "Input", rows, losses: [] }]; const snapshots: Snapshot[] = [{ name: "Input", rows, losses: [] }];
for (const stage of stages) { onProgress?.({ stage: "Input analysis", completed: 0, total: stages.length });
for (const [index, stage] of stages.entries()) {
if (!stage.enabled) { if (!stage.enabled) {
snapshots.push({ snapshots.push({
name: `${stage.type} (disabled)`, name: `${stage.type} (disabled)`,
@@ -48,6 +74,11 @@ export function runPipeline(
losses: [], losses: [],
stage, stage,
}); });
onProgress?.({
stage: `${stage.type} (disabled)`,
completed: index + 1,
total: stages.length,
});
continue; continue;
} }
const output = apply(rows, stage, right); const output = apply(rows, stage, right);
@@ -55,8 +86,18 @@ export function runPipeline(
if (rows.length > MAX_JOIN) if (rows.length > MAX_JOIN)
throw new Error(`Stage ${stage.id} expands beyond 20,000 rows.`); throw new Error(`Stage ${stage.id} expands beyond 20,000 rows.`);
snapshots.push({ name: stage.type, rows, losses: output.losses, stage }); snapshots.push({ name: stage.type, rows, losses: output.losses, stage });
onProgress?.({
stage: stage.type,
completed: index + 1,
total: stages.length,
});
} }
return { snapshots, rows, recipe: exportRecipe(stages) }; return {
snapshots,
rows,
recipe: exportRecipe(stages),
analysis: analyze(input, rows, snapshots),
};
} }
function apply( function apply(
rows: Record<string, JsonValue>[], rows: Record<string, JsonValue>[],
@@ -189,6 +230,54 @@ function apply(
losses, losses,
}; };
} }
if (stage.type === "deduplicate") {
const fields = list(c.fields);
const seen = new Set<string>();
const output = rows.filter((row) => {
const key = stableStringify(
fields.length
? Object.fromEntries(
fields.map((field) => [field, row[field] ?? null]),
)
: row,
);
if (seen.has(key)) return false;
seen.add(key);
return true;
});
return {
rows: output,
losses: [`${rows.length - output.length} duplicate rows were discarded.`],
};
}
if (stage.type === "slice") {
const offset = boundedInteger(c.offset ?? "0", "Slice offset", 0, MAX_JOIN),
count = boundedInteger(c.count ?? "100", "Slice count", 0, MAX_JOIN),
sliced = rows.slice(offset, offset + count);
return {
rows: sliced,
losses: [
`${rows.length - sliced.length} rows fall outside the selected slice.`,
],
};
}
if (stage.type === "explode") {
const field = required(c.field, "Explode field"),
target = c.target?.trim() || field,
output: Record<string, JsonValue>[] = [];
for (const row of rows) {
const value = row[field];
if (!Array.isArray(value)) {
output.push({ ...row, [target]: value ?? null });
continue;
}
for (const item of value) output.push({ ...row, [target]: item });
}
return {
rows: output,
losses: ["Array values were expanded into individual rows."],
};
}
if (stage.type === "group") { if (stage.type === "group") {
const key = required(c.key, "Group key"), const key = required(c.key, "Group key"),
fn = (c.fn ?? "count").toLowerCase(), fn = (c.fn ?? "count").toLowerCase(),
@@ -288,6 +377,7 @@ export function importRecipe(source: string): Stage[] {
throw new Error("Not a Flow Tools recipe v1."); throw new Error("Not a Flow Tools recipe v1.");
if (value.stages.length > MAX_STAGES) if (value.stages.length > MAX_STAGES)
throw new Error("Recipe exceeds 30 stages."); throw new Error("Recipe exceeds 30 stages.");
const ids = new Set<string>();
return value.stages.map((raw, index) => { return value.stages.map((raw, index) => {
if ( if (
!isObject(raw) || !isObject(raw) ||
@@ -301,23 +391,93 @@ export function importRecipe(source: string): Stage[] {
"group", "group",
"join", "join",
"format", "format",
"deduplicate",
"slice",
"explode",
].includes(raw.type) || ].includes(raw.type) ||
!raw.config || !raw.config ||
!isObject(raw.config) !isObject(raw.config)
) )
throw new Error(`Invalid recipe stage ${index + 1}.`); throw new Error(`Invalid recipe stage ${index + 1}.`);
if (raw.enabled !== undefined && typeof raw.enabled !== "boolean")
throw new Error(`Stage ${index + 1} enabled must be a boolean.`);
const id = typeof raw.id === "string" ? raw.id : `stage-${index + 1}`;
if (!/^[A-Za-z0-9_-]{1,64}$/u.test(id) || ids.has(id))
throw new Error(`Stage ${index + 1} needs a safe unique id.`);
ids.add(id);
const config: Record<string, string> = {}; const config: Record<string, string> = {};
for (const [key, item] of Object.entries(raw.config)) for (const [key, item] of Object.entries(raw.config))
if (typeof item === "string") config[key] = item; if (typeof item === "string") config[key] = item;
else throw new Error(`Stage ${index + 1} config values must be strings.`); else throw new Error(`Stage ${index + 1} config values must be strings.`);
return { return {
id: typeof raw.id === "string" ? raw.id : `stage-${index + 1}`, id,
type: raw.type as StageType, type: raw.type as StageType,
enabled: raw.enabled !== false, enabled: raw.enabled !== false,
config, config,
}; };
}); });
} }
function boundedInteger(
value: string,
label: string,
minimum: number,
maximum: number,
) {
const parsed = Number(value);
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum)
throw new Error(
`${label} must be an integer from ${minimum} to ${maximum}.`,
);
return parsed;
}
function analyze(
input: Record<string, JsonValue>[],
output: Record<string, JsonValue>[],
snapshots: Snapshot[],
): PipelineAnalysis {
const fields = new Set(output.flatMap(Object.keys));
let nullCells = 0;
for (const row of output)
for (const field of fields) if (row[field] == null) nullCells += 1;
const identities = input.map((row) => stableStringify(row));
return {
inputRows: input.length,
outputRows: output.length,
fields: fields.size,
nullCells,
duplicateRows: identities.length - new Set(identities).size,
stageRows: snapshots.map((snapshot) => snapshot.rows.length),
};
}
export function renderPipelineSvg(stages: readonly Stage[]): string {
const enabled = stages.filter((stage) => stage.enabled);
const width = Math.max(520, Math.min(2_400, 160 + enabled.length * 190));
const nodes = ["Input", ...enabled.map((stage) => stage.type), "Result"];
const body = nodes
.map((label, index) => {
const x = 20 + index * ((width - 160) / Math.max(1, nodes.length - 1));
const box = `<rect x="${x}" y="42" width="120" height="52" rx="10" fill="#f8fafc" stroke="#5b4ec4" stroke-width="2"/>`;
const text = `<text x="${x + 60}" y="74" text-anchor="middle" font-family="system-ui,sans-serif" font-size="14" fill="#202332">${escapeXml(label)}</text>`;
const previousX =
20 + (index - 1) * ((width - 160) / Math.max(1, nodes.length - 1));
const edge = index
? `<path d="M ${previousX + 120} 68 H ${x}" stroke="#64748b" stroke-width="2" marker-end="url(#arrow)"/>`
: "";
return edge + box + text;
})
.join("");
return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="136" viewBox="0 0 ${width} 136" role="img" aria-label="Flow pipeline"><defs><marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto"><path d="M0 0L10 5L0 10Z" fill="#64748b"/></marker></defs><rect width="100%" height="100%" fill="#fff"/>${body}</svg>`;
}
function escapeXml(value: string) {
return value
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;");
}
function list(value = "") { function list(value = "") {
return value return value
.split(",") .split(",")
+51
View File
@@ -82,6 +82,52 @@ button {
button:hover { button:hover {
border-color: var(--accent); border-color: var(--accent);
} }
.file-button {
display: inline-flex;
align-items: center;
border: 1px solid var(--line);
border-radius: 0.6rem;
padding: 0.55rem 0.7rem;
color: CanvasText;
cursor: pointer;
}
.file-button input {
position: absolute;
width: 1px;
height: 1px;
overflow: hidden;
clip: rect(0 0 0 0);
}
.analysis-grid {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 0.55rem;
margin: 0.7rem 0;
}
.analysis-grid div {
padding: 0.55rem;
border: 1px solid var(--line);
border-radius: 0.55rem;
}
.analysis-grid :where(dt, dd) {
margin: 0;
}
.analysis-grid dt {
color: var(--muted);
font-size: 0.75rem;
}
.analysis-grid dd {
font-weight: 700;
}
.run-actions {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
progress {
width: 100%;
accent-color: var(--accent);
}
label { label {
display: grid; display: grid;
gap: 0.25rem; gap: 0.25rem;
@@ -199,3 +245,8 @@ dialog {
flex-direction: column; flex-direction: column;
} }
} }
@media (max-width: 540px) {
.analysis-grid {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
}
+40 -2
View File
@@ -3,7 +3,7 @@
"schemaVersion": 1, "schemaVersion": 1,
"id": "de.add-ideas.flow-tools", "id": "de.add-ideas.flow-tools",
"name": "Flow Tools", "name": "Flow Tools",
"version": "0.1.0", "version": "0.2.0",
"description": "Transform structured data locally.", "description": "Transform structured data locally.",
"entry": "./", "entry": "./",
"icon": "./favicon.svg", "icon": "./favicon.svg",
@@ -16,11 +16,49 @@
}, },
"requirements": { "requirements": {
"secureContext": false, "secureContext": false,
"workers": false, "workers": true,
"indexedDb": false, "indexedDb": false,
"crossOriginIsolated": false, "crossOriginIsolated": false,
"topLevelContext": false "topLevelContext": false
}, },
"io": {
"accepts": [
{
"mediaType": "application/json",
"extensions": [".json"]
},
{
"mediaType": "application/x-ndjson",
"extensions": [".ndjson", ".jsonl"]
},
{
"mediaType": "text/csv",
"extensions": [".csv"]
},
{
"mediaType": "application/xml",
"extensions": [".xml"]
}
],
"produces": [
{
"mediaType": "application/json",
"extensions": [".json"]
},
{
"mediaType": "text/csv",
"extensions": [".csv"]
},
{
"mediaType": "image/svg+xml",
"extensions": [".svg"]
}
]
},
"capabilities": {
"required": ["workers"],
"optional": []
},
"privacy": { "privacy": {
"processing": "local", "processing": "local",
"fileUploads": true, "fileUploads": true,
+1 -1
View File
@@ -1 +1 @@
export const APP_VERSION = "0.1.0"; export const APP_VERSION = "0.2.0";
+1 -1
View File
@@ -58,6 +58,6 @@ test("PWA theme, offline reload, and manifest", async ({
const m = await request.get("/deep/nested/flow/toolbox-app.json"); const m = await request.get("/deep/nested/flow/toolbox-app.json");
await expect(m.json()).resolves.toMatchObject({ await expect(m.json()).resolves.toMatchObject({
id: "de.add-ideas.flow-tools", id: "de.add-ideas.flow-tools",
version: "0.1.0", version: "0.2.0",
}); });
}); });
+18
View File
@@ -0,0 +1,18 @@
import { expect, test } from "@playwright/test";
test("keeps the primary workspace inside a narrow viewport", async ({
page,
}) => {
await page.goto("/deep/nested/flow/");
await expect(page.locator("main").first()).toBeVisible();
await expect(
page.locator("main .loading, main .workbench-loading"),
).toHaveCount(0);
const widths = await page.evaluate(() => ({
content: document.documentElement.scrollWidth,
viewport: document.documentElement.clientWidth,
}));
expect(widths.viewport).toBeLessThanOrEqual(430);
expect(widths.content).toBeLessThanOrEqual(widths.viewport + 1);
});
+51
View File
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import { import {
importRecipe, importRecipe,
records, records,
renderPipelineSvg,
runPipeline, runPipeline,
type Stage, type Stage,
} from "../../src/core/pipeline"; } from "../../src/core/pipeline";
@@ -55,5 +56,55 @@ describe("pipeline", () => {
'{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"type":"javascript","config":{}}]}', '{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"type":"javascript","config":{}}]}',
), ),
).toThrow(/Invalid recipe/iu); ).toThrow(/Invalid recipe/iu);
expect(() =>
importRecipe(
'{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"id":"same","type":"sort","enabled":true,"config":{}},{"id":"same","type":"sort","enabled":true,"config":{}}]}',
),
).toThrow(/safe unique id/iu);
});
it("deduplicates, explodes arrays and slices deterministically", () => {
const result = runPipeline(
records([
{ id: 1, tags: ["a", "b"] },
{ id: 1, tags: ["a", "b"] },
{ id: 2, tags: ["c"] },
]),
[
{
id: "d",
type: "deduplicate",
enabled: true,
config: { fields: "id" },
},
{
id: "e",
type: "explode",
enabled: true,
config: { field: "tags", target: "tag" },
},
{
id: "s",
type: "slice",
enabled: true,
config: { offset: "1", count: "2" },
},
],
);
expect(result.rows.map((row) => row.tag)).toEqual(["b", "c"]);
expect(result.analysis).toMatchObject({
inputRows: 3,
outputRows: 2,
duplicateRows: 1,
});
expect(result.snapshots.at(-1)?.losses).toContain(
"1 rows fall outside the selected slice.",
);
});
it("exports an inert pipeline SVG", () => {
const svg = renderPipelineSvg([
{ id: "x", type: "map", enabled: true, config: {} },
]);
expect(svg).toContain("Flow pipeline");
expect(svg).not.toContain("<script");
}); });
}); });