@@ -0,0 +1,39 @@
|
||||
name: Verify
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: verify-${{ gitea.repository }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
CI: "true"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
- name: Select declared npm version
|
||||
run: npm install --global npm@11.17.0
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Audit runtime dependencies
|
||||
run: npm audit --omit=dev --audit-level=moderate
|
||||
- name: Check, test, and build
|
||||
run: npm run check
|
||||
- name: Install browser engines
|
||||
run: npx playwright install --with-deps chromium firefox webkit
|
||||
- name: Browser tests
|
||||
run: npm run test:browser
|
||||
@@ -1,5 +1,11 @@
|
||||
# Changelog
|
||||
|
||||
## 0.2.0 - 2026-09-02
|
||||
|
||||
- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue,
|
||||
explicit schema/loss evidence, cancellable worker execution and reusable
|
||||
recipe/export handoffs.
|
||||
|
||||
## 0.1.0 - 2026-09-01
|
||||
|
||||
- Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export.
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
# Flow Tools
|
||||
|
||||
Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage.
|
||||
Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted
|
||||
or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map,
|
||||
sort, group, join, type-format, deduplicate, slice and explode primitives. Every
|
||||
stage has an inspectable snapshot, dataset analysis and explicit loss notes.
|
||||
Recipes are deterministic validated JSON; there is no arbitrary code, eval,
|
||||
plugin or network stage.
|
||||
|
||||
Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`.
|
||||
Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30
|
||||
stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result
|
||||
JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`,
|
||||
`npm run check`, `npm run test:browser`, then
|
||||
`npm run package:release -- --force`.
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# Source identity
|
||||
|
||||
Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
|
||||
Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# Third-party notices
|
||||
|
||||
Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets.
|
||||
Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox
|
||||
Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0
|
||||
(GPL-3.0-or-later). Exact texts are generated in release assets.
|
||||
|
||||
+10
-1
@@ -1,3 +1,12 @@
|
||||
# Architecture
|
||||
|
||||
The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run.
|
||||
The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and
|
||||
produces JSON values. The interpreter copies rows stage-by-stage and accepts a
|
||||
fixed discriminated grammar, including deduplication, bounded slicing and array
|
||||
explosion. It records output, loss notes and dataset statistics after each run.
|
||||
|
||||
Analysis and simulation run in a disposable module worker. Progress messages
|
||||
are revision-checked, cancellation terminates the worker, and React retains the
|
||||
last successful snapshots. Recipe imports are size-bounded and validate every
|
||||
field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline
|
||||
SVG are generated entirely in the browser.
|
||||
|
||||
Generated
+20
-20
@@ -1,23 +1,23 @@
|
||||
{
|
||||
"name": "flow-tools",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "flow-tools",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3",
|
||||
"@add-ideas/toolbox-helpers": "0.1.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.2.3",
|
||||
"@add-ideas/toolbox-contract": "0.3.0",
|
||||
"@add-ideas/toolbox-helpers": "0.2.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.3.0",
|
||||
"@xmldom/xmldom": "0.9.12",
|
||||
"react": "19.2.8",
|
||||
"react-dom": "19.2.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@add-ideas/toolbox-testkit": "0.2.3",
|
||||
"@add-ideas/toolbox-testkit": "0.3.0",
|
||||
"@eslint/js": "10.0.1",
|
||||
"@playwright/test": "1.62.1",
|
||||
"@testing-library/jest-dom": "6.9.1",
|
||||
@@ -43,24 +43,24 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-contract": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.2.3/toolbox-contract-0.2.3.tgz",
|
||||
"integrity": "sha512-T0PVSuMT40GjTDfQJhEEY3ZawQq8zz1/ry95JdKI6W39CdLacaRXdGnEpDCMHt+jUbf1Jz7Nat/M5dFCgKVM9A==",
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz",
|
||||
"integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-helpers": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.1.0/toolbox-helpers-0.1.0.tgz",
|
||||
"integrity": "sha512-UKl1Oxekedf8D2df86VrnVA53AcMhrnh6iUPXY+k8frirBXotb0yd8SGT+IF/3hcqYwcYe/v9WVFuSgKtIYVnw==",
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-helpers/-/0.2.0/toolbox-helpers-0.2.0.tgz",
|
||||
"integrity": "sha512-SdOqkw+P+3J3fa5iVkzb5P15rVepB001GNV21Oh8w0CZcVL+YRltgD/s+MVcTyrNijWQf3E5vtQON/3N2LLyKg==",
|
||||
"license": "GPL-3.0-or-later"
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-shell-react": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.2.3/toolbox-shell-react-0.2.3.tgz",
|
||||
"integrity": "sha512-DT5lQDH48BFkFcmFLZnQh7+Cm73JzBPcmp5WzUXypfkUXpEyDYHzaXgmW4kZ0edSwh4RK4sPmx+JPtK0X4aKCQ==",
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz",
|
||||
"integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3"
|
||||
"@add-ideas/toolbox-contract": "0.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=18 <20",
|
||||
@@ -68,13 +68,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-testkit": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.2.3/toolbox-testkit-0.2.3.tgz",
|
||||
"integrity": "sha512-sq1MwhKWfFKen+N+124hl74qQimRSvmQ9sOU7jdcI+2qCKZ67+2B8rWyezeV80uTFu4Jv6deHksfYQ/tKNV6XQ==",
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz",
|
||||
"integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3"
|
||||
"@add-ideas/toolbox-contract": "0.3.0"
|
||||
},
|
||||
"bin": {
|
||||
"toolbox-check": "dist/cli.js"
|
||||
|
||||
+8
-5
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "flow-tools",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"description": "Build and inspect bounded local data transformation pipelines.",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"author": "Albrecht Degering",
|
||||
@@ -9,6 +9,9 @@
|
||||
"url": "git+https://git.add-ideas.de/lotobo/flow-tools.git"
|
||||
},
|
||||
"homepage": "https://git.add-ideas.de/lotobo/flow-tools",
|
||||
"bugs": {
|
||||
"url": "https://git.add-ideas.de/lotobo/flow-tools/issues"
|
||||
},
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
@@ -34,15 +37,15 @@
|
||||
"release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force"
|
||||
},
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3",
|
||||
"@add-ideas/toolbox-helpers": "0.1.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.2.3",
|
||||
"@add-ideas/toolbox-contract": "0.3.0",
|
||||
"@add-ideas/toolbox-helpers": "0.2.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.3.0",
|
||||
"@xmldom/xmldom": "0.9.12",
|
||||
"react": "19.2.8",
|
||||
"react-dom": "19.2.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@add-ideas/toolbox-testkit": "0.2.3",
|
||||
"@add-ideas/toolbox-testkit": "0.3.0",
|
||||
"@eslint/js": "10.0.1",
|
||||
"@playwright/test": "1.62.1",
|
||||
"@testing-library/jest-dom": "6.9.1",
|
||||
|
||||
+20
-2
@@ -12,7 +12,25 @@ export default defineConfig({
|
||||
timeout: 180000,
|
||||
},
|
||||
projects: [
|
||||
{ name: "chromium", use: { ...devices["Desktop Chrome"] } },
|
||||
{ name: "firefox", use: { ...devices["Desktop Firefox"] } },
|
||||
{
|
||||
name: "chromium",
|
||||
testIgnore: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Desktop Chrome"] },
|
||||
},
|
||||
{
|
||||
name: "firefox",
|
||||
testIgnore: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Desktop Firefox"] },
|
||||
},
|
||||
{
|
||||
name: "webkit",
|
||||
testIgnore: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Desktop Safari"] },
|
||||
},
|
||||
{
|
||||
name: "mobile-chromium",
|
||||
testMatch: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Pixel 5"] },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# Changelog
|
||||
|
||||
## 0.2.0 - 2026-09-02
|
||||
|
||||
- Add bounded CSV/JSON/NDJSON/XML import, a richer transformation catalogue,
|
||||
explicit schema/loss evidence, cancellable worker execution and reusable
|
||||
recipe/export handoffs.
|
||||
|
||||
## 0.1.0 - 2026-09-01
|
||||
|
||||
- Add eight bounded transformation primitives, stage snapshots/losses, joins and deterministic recipe/result export.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
==============================================================================
|
||||
@add-ideas/toolbox-contract@0.2.3
|
||||
@add-ideas/toolbox-contract@0.3.0
|
||||
Declared licence: Apache-2.0
|
||||
==============================================================================
|
||||
--- LICENSE ---
|
||||
@@ -198,7 +198,7 @@ Declared licence: Apache-2.0
|
||||
|
||||
|
||||
==============================================================================
|
||||
@add-ideas/toolbox-helpers@0.1.0
|
||||
@add-ideas/toolbox-helpers@0.2.0
|
||||
Declared licence: GPL-3.0-or-later
|
||||
==============================================================================
|
||||
--- LICENSE ---
|
||||
@@ -879,7 +879,7 @@ Public License instead of this License. But first, please read
|
||||
|
||||
|
||||
==============================================================================
|
||||
@add-ideas/toolbox-shell-react@0.2.3
|
||||
@add-ideas/toolbox-shell-react@0.3.0
|
||||
Declared licence: Apache-2.0
|
||||
==============================================================================
|
||||
--- LICENSE ---
|
||||
|
||||
+11
-2
@@ -1,5 +1,14 @@
|
||||
# Flow Tools
|
||||
|
||||
Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It parses bounded JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map, sort, group, join and type-format primitives. Every stage has an inspectable snapshot and explicit loss notes. Recipes are deterministic validated JSON; there is no arbitrary code, eval, plugin or network stage.
|
||||
Flow Tools is a GPL-3.0-or-later local data-pipeline designer. It accepts pasted
|
||||
or local JSON/CSV/NDJSON/XML and composes fixed select, rename, filter, map,
|
||||
sort, group, join, type-format, deduplicate, slice and explode primitives. Every
|
||||
stage has an inspectable snapshot, dataset analysis and explicit loss notes.
|
||||
Recipes are deterministic validated JSON; there is no arbitrary code, eval,
|
||||
plugin or network stage.
|
||||
|
||||
Limits: 2 MiB sources, 10,000 input rows, 20,000 join output rows, 200 fields, 30 stages and 256 KiB recipe JSON. Result JSON/CSV and recipes export explicitly. Run `npm ci`, `npm run check`, `npm run test:browser`, then `npm run package:release -- --force`.
|
||||
Limits: 2 MiB sources, 10,000 input rows, 20,000 expanded rows, 200 fields, 30
|
||||
stages and 256 KiB recipe JSON. Simulation runs in a cancellable worker. Result
|
||||
JSON/CSV, recipes and an inert pipeline SVG export explicitly. Run `npm ci`,
|
||||
`npm run check`, `npm run test:browser`, then
|
||||
`npm run package:release -- --force`.
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
# Source identity
|
||||
|
||||
Flow Tools 0.1.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
|
||||
Flow Tools 0.2.0 · `de.add-ideas.flow-tools` · https://git.add-ideas.de/lotobo/flow-tools · GPL-3.0-or-later. Release order, timestamps and permissions are normalized with a SHA-256 sidecar.
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# Third-party notices
|
||||
|
||||
Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox packages (GPL-3.0-or-later). Exact texts are generated in release assets.
|
||||
Bundled runtime: React/React DOM and @xmldom/xmldom (MIT); add·ideas Toolbox
|
||||
Contract and Shell React 0.3.0 (Apache-2.0); and Toolbox Helpers 0.2.0
|
||||
(GPL-3.0-or-later). Exact texts are generated in release assets.
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
# Architecture
|
||||
|
||||
The bounded parser produces JSON values. The interpreter copies rows stage-by-stage and accepts a fixed discriminated stage grammar. It records output and loss notes after each stage. Recipe import validates every field before execution; React retains the last successful run.
|
||||
The bounded parser accepts pasted or local JSON, CSV, NDJSON and inert XML and
|
||||
produces JSON values. The interpreter copies rows stage-by-stage and accepts a
|
||||
fixed discriminated grammar, including deduplication, bounded slicing and array
|
||||
explosion. It records output, loss notes and dataset statistics after each run.
|
||||
|
||||
Analysis and simulation run in a disposable module worker. Progress messages
|
||||
are revision-checked, cancellation terminates the worker, and React retains the
|
||||
last successful snapshots. Recipe imports are size-bounded and validate every
|
||||
field before execution. Result JSON/CSV, recipe JSON and the escaped pipeline
|
||||
SVG are generated entirely in the browser.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
const CACHE = "flow-tools-v0.1.0",
|
||||
const CACHE = "flow-tools-v0.2.0",
|
||||
APP = [
|
||||
"./",
|
||||
"./index.html",
|
||||
|
||||
+19
-2
@@ -3,7 +3,7 @@
|
||||
"schemaVersion": 1,
|
||||
"id": "de.add-ideas.flow-tools",
|
||||
"name": "Flow Tools",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"description": "Transform structured data locally.",
|
||||
"entry": "./",
|
||||
"icon": "./favicon.svg",
|
||||
@@ -16,11 +16,28 @@
|
||||
},
|
||||
"requirements": {
|
||||
"secureContext": false,
|
||||
"workers": false,
|
||||
"workers": true,
|
||||
"indexedDb": false,
|
||||
"crossOriginIsolated": false,
|
||||
"topLevelContext": false
|
||||
},
|
||||
"io": {
|
||||
"accepts": [
|
||||
{ "mediaType": "application/json", "extensions": [".json"] },
|
||||
{
|
||||
"mediaType": "application/x-ndjson",
|
||||
"extensions": [".ndjson", ".jsonl"]
|
||||
},
|
||||
{ "mediaType": "text/csv", "extensions": [".csv"] },
|
||||
{ "mediaType": "application/xml", "extensions": [".xml"] }
|
||||
],
|
||||
"produces": [
|
||||
{ "mediaType": "application/json", "extensions": [".json"] },
|
||||
{ "mediaType": "text/csv", "extensions": [".csv"] },
|
||||
{ "mediaType": "image/svg+xml", "extensions": [".svg"] }
|
||||
]
|
||||
},
|
||||
"capabilities": { "required": ["workers"], "optional": [] },
|
||||
"privacy": {
|
||||
"processing": "local",
|
||||
"fileUploads": true,
|
||||
|
||||
@@ -18,7 +18,12 @@ export function HelpDialog({
|
||||
onCancel={onClose}
|
||||
aria-label="About Flow Tools"
|
||||
>
|
||||
<button className="close" onClick={onClose}>
|
||||
<button
|
||||
type="button"
|
||||
className="close"
|
||||
aria-label="Close help"
|
||||
onClick={onClose}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
<h2>Flow Tools</h2>
|
||||
|
||||
+214
-10
@@ -1,4 +1,4 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { useMemo, useRef, useState } from "react";
|
||||
import {
|
||||
stableStringify,
|
||||
stringifyCsv,
|
||||
@@ -6,10 +6,12 @@ import {
|
||||
type JsonValue,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import { parseData, type DataFormat } from "../core/data";
|
||||
import { runFlowJob } from "../core/flow-client";
|
||||
import {
|
||||
exportRecipe,
|
||||
importRecipe,
|
||||
records,
|
||||
renderPipelineSvg,
|
||||
runPipeline,
|
||||
type PipelineResult,
|
||||
type Stage,
|
||||
@@ -48,7 +50,12 @@ export function Workbench() {
|
||||
),
|
||||
[selected, setSelected] = useState(2),
|
||||
[status, setStatus] = useState("Sample pipeline completed locally."),
|
||||
[recipeText, setRecipeText] = useState("");
|
||||
[recipeText, setRecipeText] = useState(""),
|
||||
[busy, setBusy] = useState(false),
|
||||
[progress, setProgress] = useState(0);
|
||||
const operation = useRef<{ revision: number; controller?: AbortController }>({
|
||||
revision: 0,
|
||||
});
|
||||
const snapshot =
|
||||
result.snapshots[Math.min(selected, result.snapshots.length - 1)]!;
|
||||
function parsePrimary() {
|
||||
@@ -71,16 +78,42 @@ export function Workbench() {
|
||||
setStatus(`${msg(e)} Last valid secondary input retained.`);
|
||||
}
|
||||
}
|
||||
function run() {
|
||||
async function run() {
|
||||
operation.current.controller?.abort();
|
||||
const controller = new AbortController(),
|
||||
revision = ++operation.current.revision;
|
||||
operation.current.controller = controller;
|
||||
setBusy(true);
|
||||
setProgress(0);
|
||||
try {
|
||||
const next = runPipeline(input, stages, right);
|
||||
const next = await runFlowJob(input, stages, right, {
|
||||
signal: controller.signal,
|
||||
onProgress: (item) => {
|
||||
if (operation.current.revision !== revision) return;
|
||||
setProgress(item.total ? item.completed / item.total : 0);
|
||||
setStatus(`Running ${item.stage} locally…`);
|
||||
},
|
||||
});
|
||||
if (operation.current.revision !== revision) return;
|
||||
setResult(next);
|
||||
setSelected(next.snapshots.length - 1);
|
||||
setStatus(
|
||||
`Completed ${stages.filter((s) => s.enabled).length} stages; ${next.rows.length} rows remain.`,
|
||||
);
|
||||
} catch (e) {
|
||||
if (operation.current.revision !== revision) return;
|
||||
if (e instanceof DOMException && e.name === "AbortError") {
|
||||
setStatus(
|
||||
"Pipeline run cancelled. Last successful snapshots remain visible.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
setStatus(`${msg(e)} Last successful stage snapshots remain visible.`);
|
||||
} finally {
|
||||
if (operation.current.revision === revision) {
|
||||
setBusy(false);
|
||||
operation.current.controller = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
function update(index: number, stage: Stage) {
|
||||
@@ -90,19 +123,24 @@ export function Workbench() {
|
||||
setStages((s) => [
|
||||
...s,
|
||||
{
|
||||
id: `stage-${Date.now().toString(36)}`,
|
||||
id: nextStageId(s),
|
||||
type,
|
||||
enabled: true,
|
||||
config: defaults(type),
|
||||
},
|
||||
]);
|
||||
}
|
||||
function download(kind: "json" | "csv" | "recipe") {
|
||||
function download(kind: "json" | "csv" | "recipe" | "svg") {
|
||||
if (kind === "recipe")
|
||||
triggerBlobDownload(
|
||||
new Blob([exportRecipe(stages)], { type: "application/json" }),
|
||||
"flow-recipe.json",
|
||||
);
|
||||
else if (kind === "svg")
|
||||
triggerBlobDownload(
|
||||
new Blob([renderPipelineSvg(stages)], { type: "image/svg+xml" }),
|
||||
"flow-pipeline.svg",
|
||||
);
|
||||
else if (kind === "json")
|
||||
triggerBlobDownload(
|
||||
new Blob([stableStringify(result.rows, 2)], {
|
||||
@@ -124,6 +162,41 @@ export function Workbench() {
|
||||
);
|
||||
}
|
||||
}
|
||||
async function openData(file: File | undefined, secondary = false) {
|
||||
if (!file) return;
|
||||
try {
|
||||
if (file.size > 2 * 1024 * 1024)
|
||||
throw new Error("Input file exceeds the 2 MiB limit.");
|
||||
const text = await file.text(),
|
||||
detected = inferFormat(file.name),
|
||||
next = records(parseData(text, detected).rows);
|
||||
if (secondary) {
|
||||
setRightSource(text);
|
||||
setRight(next);
|
||||
setStatus(`Opened ${next.length} secondary rows from ${file.name}.`);
|
||||
} else {
|
||||
setFormat(detected);
|
||||
setSource(text);
|
||||
setInput(next);
|
||||
setStatus(`Opened ${next.length} primary rows from ${file.name}.`);
|
||||
}
|
||||
} catch (e) {
|
||||
setStatus(`${msg(e)} Last valid input retained.`);
|
||||
}
|
||||
}
|
||||
async function openRecipe(file: File | undefined) {
|
||||
if (!file) return;
|
||||
try {
|
||||
if (file.size > 256 * 1024) throw new Error("Recipe exceeds 256 KiB.");
|
||||
const text = await file.text(),
|
||||
next = importRecipe(text);
|
||||
setRecipeText(text);
|
||||
setStages(next);
|
||||
setStatus(`Imported ${next.length} validated stages from ${file.name}.`);
|
||||
} catch (e) {
|
||||
setStatus(msg(e));
|
||||
}
|
||||
}
|
||||
function loadRecipe() {
|
||||
try {
|
||||
const next = importRecipe(recipeText);
|
||||
@@ -144,11 +217,25 @@ export function Workbench() {
|
||||
explicit losses with a deterministic recipe.
|
||||
</p>
|
||||
</div>
|
||||
<button className="primary" onClick={run}>
|
||||
<div className="run-actions">
|
||||
<button
|
||||
className="primary"
|
||||
onClick={() => void run()}
|
||||
disabled={busy}
|
||||
>
|
||||
Run complete pipeline
|
||||
</button>
|
||||
{busy && (
|
||||
<button onClick={() => operation.current.controller?.abort()}>
|
||||
Cancel
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
<p role="status">{status}</p>
|
||||
{busy && (
|
||||
<progress value={progress} max={1} aria-label="Pipeline progress" />
|
||||
)}
|
||||
<section className="sources">
|
||||
<section className="panel">
|
||||
<h2>Primary source</h2>
|
||||
@@ -169,7 +256,17 @@ export function Workbench() {
|
||||
onChange={(e) => setSource(e.target.value)}
|
||||
aria-label="Primary source"
|
||||
/>
|
||||
<div className="exports">
|
||||
<button onClick={parsePrimary}>Parse primary</button>
|
||||
<label className="file-button">
|
||||
Open data file
|
||||
<input
|
||||
type="file"
|
||||
accept=".json,.csv,.ndjson,.jsonl,.xml"
|
||||
onChange={(event) => void openData(event.target.files?.[0])}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</section>
|
||||
<section className="panel">
|
||||
<h2>Secondary join source</h2>
|
||||
@@ -179,7 +276,19 @@ export function Workbench() {
|
||||
onChange={(e) => setRightSource(e.target.value)}
|
||||
aria-label="Secondary source"
|
||||
/>
|
||||
<div className="exports">
|
||||
<button onClick={parseRight}>Parse secondary</button>
|
||||
<label className="file-button">
|
||||
Open join file
|
||||
<input
|
||||
type="file"
|
||||
accept=".json,.csv,.ndjson,.jsonl,.xml"
|
||||
onChange={(event) =>
|
||||
void openData(event.target.files?.[0], true)
|
||||
}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</section>
|
||||
</section>
|
||||
<section className="pipeline">
|
||||
@@ -196,6 +305,9 @@ export function Workbench() {
|
||||
<option value="group">Group</option>
|
||||
<option value="join">Join</option>
|
||||
<option value="format">Format type</option>
|
||||
<option value="deduplicate">Remove duplicates</option>
|
||||
<option value="slice">Slice rows</option>
|
||||
<option value="explode">Explode array</option>
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
@@ -250,10 +362,31 @@ export function Workbench() {
|
||||
</ul>
|
||||
)}
|
||||
<Table rows={snapshot.rows} />
|
||||
<dl className="analysis-grid" aria-label="Result analysis">
|
||||
<div>
|
||||
<dt>Rows</dt>
|
||||
<dd>
|
||||
{result.analysis.inputRows} → {result.analysis.outputRows}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Fields</dt>
|
||||
<dd>{result.analysis.fields}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Empty cells</dt>
|
||||
<dd>{result.analysis.nullCells}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Input duplicates</dt>
|
||||
<dd>{result.analysis.duplicateRows}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<div className="exports">
|
||||
<button onClick={() => download("json")}>Result JSON</button>
|
||||
<button onClick={() => download("csv")}>Result CSV</button>
|
||||
<button onClick={() => download("recipe")}>Recipe JSON</button>
|
||||
<button onClick={() => download("svg")}>Pipeline SVG</button>
|
||||
</div>
|
||||
<details>
|
||||
<summary>Import deterministic recipe</summary>
|
||||
@@ -263,6 +396,14 @@ export function Workbench() {
|
||||
aria-label="Recipe JSON"
|
||||
/>
|
||||
<button onClick={loadRecipe}>Validate and import</button>
|
||||
<label className="file-button">
|
||||
Open recipe file
|
||||
<input
|
||||
type="file"
|
||||
accept=".json,application/json"
|
||||
onChange={(event) => void openRecipe(event.target.files?.[0])}
|
||||
/>
|
||||
</label>
|
||||
</details>
|
||||
</section>
|
||||
</section>
|
||||
@@ -303,8 +444,20 @@ function StageEditor({
|
||||
</header>
|
||||
<Config stage={stage} set={set} />
|
||||
<footer>
|
||||
<button onClick={() => move(-1)}>↑</button>
|
||||
<button onClick={() => move(1)}>↓</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Move stage up"
|
||||
onClick={() => move(-1)}
|
||||
>
|
||||
↑
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Move stage down"
|
||||
onClick={() => move(1)}
|
||||
>
|
||||
↓
|
||||
</button>
|
||||
<button onClick={remove}>Remove</button>
|
||||
</footer>
|
||||
</article>
|
||||
@@ -415,6 +568,36 @@ function Config({
|
||||
/>
|
||||
</>
|
||||
);
|
||||
if (stage.type === "deduplicate")
|
||||
return (
|
||||
<Field
|
||||
label="Identity fields (blank = whole row)"
|
||||
value={f.fields}
|
||||
set={(v) => set("fields", v)}
|
||||
/>
|
||||
);
|
||||
if (stage.type === "slice")
|
||||
return (
|
||||
<>
|
||||
<Field label="Offset" value={f.offset} set={(v) => set("offset", v)} />
|
||||
<Field label="Count" value={f.count} set={(v) => set("count", v)} />
|
||||
</>
|
||||
);
|
||||
if (stage.type === "explode")
|
||||
return (
|
||||
<>
|
||||
<Field
|
||||
label="Array field"
|
||||
value={f.field}
|
||||
set={(v) => set("field", v)}
|
||||
/>
|
||||
<Field
|
||||
label="Output field"
|
||||
value={f.target}
|
||||
set={(v) => set("target", v)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
return (
|
||||
<>
|
||||
<Field label="Target" value={f.target} set={(v) => set("target", v)} />
|
||||
@@ -530,5 +713,26 @@ function defaults(t: StageType): Record<string, string> {
|
||||
mode: "left",
|
||||
prefix: "right_",
|
||||
}
|
||||
: { field: "score", as: "number" };
|
||||
: t === "format"
|
||||
? { field: "score", as: "number" }
|
||||
: t === "deduplicate"
|
||||
? { fields: "" }
|
||||
: t === "slice"
|
||||
? { offset: "0", count: "100" }
|
||||
: { field: "items", target: "item" };
|
||||
}
|
||||
|
||||
function inferFormat(fileName: string): DataFormat {
|
||||
const extension = fileName.split(".").at(-1)?.toLowerCase();
|
||||
if (extension === "csv") return "csv";
|
||||
if (extension === "ndjson" || extension === "jsonl") return "ndjson";
|
||||
if (extension === "xml") return "xml";
|
||||
return "json";
|
||||
}
|
||||
|
||||
function nextStageId(stages: readonly Stage[]): string {
|
||||
const ids = new Set(stages.map((stage) => stage.id));
|
||||
let suffix = 1;
|
||||
while (ids.has(`stage-${suffix}`)) suffix += 1;
|
||||
return `stage-${suffix}`;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { JsonValue } from "@add-ideas/toolbox-helpers";
|
||||
import {
|
||||
runPipeline,
|
||||
type PipelineProgress,
|
||||
type PipelineResult,
|
||||
type Stage,
|
||||
} from "./pipeline";
|
||||
|
||||
export async function runFlowJob(
|
||||
input: Record<string, JsonValue>[],
|
||||
stages: Stage[],
|
||||
right: Record<string, JsonValue>[],
|
||||
options: {
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (progress: PipelineProgress) => void;
|
||||
} = {},
|
||||
): Promise<PipelineResult> {
|
||||
if (options.signal?.aborted) throw abortError();
|
||||
if (typeof Worker === "undefined")
|
||||
return runPipeline(input, stages, right, options.onProgress);
|
||||
const worker = new Worker(new URL("./flow.worker.ts", import.meta.url), {
|
||||
type: "module",
|
||||
name: "flow-tools-analysis",
|
||||
});
|
||||
const id = Date.now() + Math.random();
|
||||
return await new Promise<PipelineResult>((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
options.signal?.removeEventListener("abort", cancel);
|
||||
worker.terminate();
|
||||
};
|
||||
const cancel = () => {
|
||||
cleanup();
|
||||
reject(abortError());
|
||||
};
|
||||
options.signal?.addEventListener("abort", cancel, { once: true });
|
||||
worker.addEventListener("message", (event) => {
|
||||
if (event.data?.id !== id) return;
|
||||
if (event.data.type === "progress") {
|
||||
options.onProgress?.(event.data.progress as PipelineProgress);
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
if (event.data.type === "result")
|
||||
resolve(event.data.result as PipelineResult);
|
||||
else reject(new Error(String(event.data.error ?? "Flow worker failed.")));
|
||||
});
|
||||
worker.addEventListener("error", () => {
|
||||
cleanup();
|
||||
reject(new Error("The isolated flow worker stopped unexpectedly."));
|
||||
});
|
||||
worker.postMessage({ id, input, stages, right });
|
||||
});
|
||||
}
|
||||
|
||||
function abortError() {
|
||||
return new DOMException("Flow operation cancelled.", "AbortError");
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/// <reference lib="webworker" />
|
||||
import { runPipeline, type PipelineProgress, type Stage } from "./pipeline";
|
||||
import type { JsonValue } from "@add-ideas/toolbox-helpers";
|
||||
|
||||
interface Request {
|
||||
id: number;
|
||||
input: Record<string, JsonValue>[];
|
||||
right: Record<string, JsonValue>[];
|
||||
stages: Stage[];
|
||||
}
|
||||
|
||||
const scope = self as DedicatedWorkerGlobalScope;
|
||||
scope.addEventListener("message", (event: MessageEvent<Request>) => {
|
||||
const request = event.data;
|
||||
try {
|
||||
const result = runPipeline(
|
||||
request.input,
|
||||
request.stages,
|
||||
request.right,
|
||||
(progress: PipelineProgress) =>
|
||||
scope.postMessage({ id: request.id, type: "progress", progress }),
|
||||
);
|
||||
scope.postMessage({ id: request.id, type: "result", result });
|
||||
} catch (error) {
|
||||
scope.postMessage({
|
||||
id: request.id,
|
||||
type: "error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
+164
-4
@@ -5,7 +5,17 @@ import {
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import { isObject } from "./data";
|
||||
export type StageType =
|
||||
"select" | "rename" | "filter" | "map" | "sort" | "group" | "join" | "format";
|
||||
| "select"
|
||||
| "rename"
|
||||
| "filter"
|
||||
| "map"
|
||||
| "sort"
|
||||
| "group"
|
||||
| "join"
|
||||
| "format"
|
||||
| "deduplicate"
|
||||
| "slice"
|
||||
| "explode";
|
||||
export interface Stage {
|
||||
id: string;
|
||||
type: StageType;
|
||||
@@ -22,6 +32,20 @@ export interface PipelineResult {
|
||||
snapshots: Snapshot[];
|
||||
rows: Record<string, JsonValue>[];
|
||||
recipe: string;
|
||||
analysis: PipelineAnalysis;
|
||||
}
|
||||
export interface PipelineAnalysis {
|
||||
inputRows: number;
|
||||
outputRows: number;
|
||||
fields: number;
|
||||
nullCells: number;
|
||||
duplicateRows: number;
|
||||
stageRows: number[];
|
||||
}
|
||||
export interface PipelineProgress {
|
||||
stage: string;
|
||||
completed: number;
|
||||
total: number;
|
||||
}
|
||||
const MAX_STAGES = 30,
|
||||
MAX_ROWS = 10000,
|
||||
@@ -36,11 +60,13 @@ export function runPipeline(
|
||||
input: Record<string, JsonValue>[],
|
||||
stages: Stage[],
|
||||
right: Record<string, JsonValue>[] = [],
|
||||
onProgress?: (progress: PipelineProgress) => void,
|
||||
): PipelineResult {
|
||||
if (stages.length > MAX_STAGES) throw new Error("Recipe exceeds 30 stages.");
|
||||
let rows = input.map((row) => ({ ...row }));
|
||||
const snapshots: Snapshot[] = [{ name: "Input", rows, losses: [] }];
|
||||
for (const stage of stages) {
|
||||
onProgress?.({ stage: "Input analysis", completed: 0, total: stages.length });
|
||||
for (const [index, stage] of stages.entries()) {
|
||||
if (!stage.enabled) {
|
||||
snapshots.push({
|
||||
name: `${stage.type} (disabled)`,
|
||||
@@ -48,6 +74,11 @@ export function runPipeline(
|
||||
losses: [],
|
||||
stage,
|
||||
});
|
||||
onProgress?.({
|
||||
stage: `${stage.type} (disabled)`,
|
||||
completed: index + 1,
|
||||
total: stages.length,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const output = apply(rows, stage, right);
|
||||
@@ -55,8 +86,18 @@ export function runPipeline(
|
||||
if (rows.length > MAX_JOIN)
|
||||
throw new Error(`Stage ${stage.id} expands beyond 20,000 rows.`);
|
||||
snapshots.push({ name: stage.type, rows, losses: output.losses, stage });
|
||||
onProgress?.({
|
||||
stage: stage.type,
|
||||
completed: index + 1,
|
||||
total: stages.length,
|
||||
});
|
||||
}
|
||||
return { snapshots, rows, recipe: exportRecipe(stages) };
|
||||
return {
|
||||
snapshots,
|
||||
rows,
|
||||
recipe: exportRecipe(stages),
|
||||
analysis: analyze(input, rows, snapshots),
|
||||
};
|
||||
}
|
||||
function apply(
|
||||
rows: Record<string, JsonValue>[],
|
||||
@@ -189,6 +230,54 @@ function apply(
|
||||
losses,
|
||||
};
|
||||
}
|
||||
if (stage.type === "deduplicate") {
|
||||
const fields = list(c.fields);
|
||||
const seen = new Set<string>();
|
||||
const output = rows.filter((row) => {
|
||||
const key = stableStringify(
|
||||
fields.length
|
||||
? Object.fromEntries(
|
||||
fields.map((field) => [field, row[field] ?? null]),
|
||||
)
|
||||
: row,
|
||||
);
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
return {
|
||||
rows: output,
|
||||
losses: [`${rows.length - output.length} duplicate rows were discarded.`],
|
||||
};
|
||||
}
|
||||
if (stage.type === "slice") {
|
||||
const offset = boundedInteger(c.offset ?? "0", "Slice offset", 0, MAX_JOIN),
|
||||
count = boundedInteger(c.count ?? "100", "Slice count", 0, MAX_JOIN),
|
||||
sliced = rows.slice(offset, offset + count);
|
||||
return {
|
||||
rows: sliced,
|
||||
losses: [
|
||||
`${rows.length - sliced.length} rows fall outside the selected slice.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
if (stage.type === "explode") {
|
||||
const field = required(c.field, "Explode field"),
|
||||
target = c.target?.trim() || field,
|
||||
output: Record<string, JsonValue>[] = [];
|
||||
for (const row of rows) {
|
||||
const value = row[field];
|
||||
if (!Array.isArray(value)) {
|
||||
output.push({ ...row, [target]: value ?? null });
|
||||
continue;
|
||||
}
|
||||
for (const item of value) output.push({ ...row, [target]: item });
|
||||
}
|
||||
return {
|
||||
rows: output,
|
||||
losses: ["Array values were expanded into individual rows."],
|
||||
};
|
||||
}
|
||||
if (stage.type === "group") {
|
||||
const key = required(c.key, "Group key"),
|
||||
fn = (c.fn ?? "count").toLowerCase(),
|
||||
@@ -288,6 +377,7 @@ export function importRecipe(source: string): Stage[] {
|
||||
throw new Error("Not a Flow Tools recipe v1.");
|
||||
if (value.stages.length > MAX_STAGES)
|
||||
throw new Error("Recipe exceeds 30 stages.");
|
||||
const ids = new Set<string>();
|
||||
return value.stages.map((raw, index) => {
|
||||
if (
|
||||
!isObject(raw) ||
|
||||
@@ -301,23 +391,93 @@ export function importRecipe(source: string): Stage[] {
|
||||
"group",
|
||||
"join",
|
||||
"format",
|
||||
"deduplicate",
|
||||
"slice",
|
||||
"explode",
|
||||
].includes(raw.type) ||
|
||||
!raw.config ||
|
||||
!isObject(raw.config)
|
||||
)
|
||||
throw new Error(`Invalid recipe stage ${index + 1}.`);
|
||||
if (raw.enabled !== undefined && typeof raw.enabled !== "boolean")
|
||||
throw new Error(`Stage ${index + 1} enabled must be a boolean.`);
|
||||
const id = typeof raw.id === "string" ? raw.id : `stage-${index + 1}`;
|
||||
if (!/^[A-Za-z0-9_-]{1,64}$/u.test(id) || ids.has(id))
|
||||
throw new Error(`Stage ${index + 1} needs a safe unique id.`);
|
||||
ids.add(id);
|
||||
const config: Record<string, string> = {};
|
||||
for (const [key, item] of Object.entries(raw.config))
|
||||
if (typeof item === "string") config[key] = item;
|
||||
else throw new Error(`Stage ${index + 1} config values must be strings.`);
|
||||
return {
|
||||
id: typeof raw.id === "string" ? raw.id : `stage-${index + 1}`,
|
||||
id,
|
||||
type: raw.type as StageType,
|
||||
enabled: raw.enabled !== false,
|
||||
config,
|
||||
};
|
||||
});
|
||||
}
|
||||
function boundedInteger(
|
||||
value: string,
|
||||
label: string,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum)
|
||||
throw new Error(
|
||||
`${label} must be an integer from ${minimum} to ${maximum}.`,
|
||||
);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function analyze(
|
||||
input: Record<string, JsonValue>[],
|
||||
output: Record<string, JsonValue>[],
|
||||
snapshots: Snapshot[],
|
||||
): PipelineAnalysis {
|
||||
const fields = new Set(output.flatMap(Object.keys));
|
||||
let nullCells = 0;
|
||||
for (const row of output)
|
||||
for (const field of fields) if (row[field] == null) nullCells += 1;
|
||||
const identities = input.map((row) => stableStringify(row));
|
||||
return {
|
||||
inputRows: input.length,
|
||||
outputRows: output.length,
|
||||
fields: fields.size,
|
||||
nullCells,
|
||||
duplicateRows: identities.length - new Set(identities).size,
|
||||
stageRows: snapshots.map((snapshot) => snapshot.rows.length),
|
||||
};
|
||||
}
|
||||
|
||||
export function renderPipelineSvg(stages: readonly Stage[]): string {
|
||||
const enabled = stages.filter((stage) => stage.enabled);
|
||||
const width = Math.max(520, Math.min(2_400, 160 + enabled.length * 190));
|
||||
const nodes = ["Input", ...enabled.map((stage) => stage.type), "Result"];
|
||||
const body = nodes
|
||||
.map((label, index) => {
|
||||
const x = 20 + index * ((width - 160) / Math.max(1, nodes.length - 1));
|
||||
const box = `<rect x="${x}" y="42" width="120" height="52" rx="10" fill="#f8fafc" stroke="#5b4ec4" stroke-width="2"/>`;
|
||||
const text = `<text x="${x + 60}" y="74" text-anchor="middle" font-family="system-ui,sans-serif" font-size="14" fill="#202332">${escapeXml(label)}</text>`;
|
||||
const previousX =
|
||||
20 + (index - 1) * ((width - 160) / Math.max(1, nodes.length - 1));
|
||||
const edge = index
|
||||
? `<path d="M ${previousX + 120} 68 H ${x}" stroke="#64748b" stroke-width="2" marker-end="url(#arrow)"/>`
|
||||
: "";
|
||||
return edge + box + text;
|
||||
})
|
||||
.join("");
|
||||
return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="136" viewBox="0 0 ${width} 136" role="img" aria-label="Flow pipeline"><defs><marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto"><path d="M0 0L10 5L0 10Z" fill="#64748b"/></marker></defs><rect width="100%" height="100%" fill="#fff"/>${body}</svg>`;
|
||||
}
|
||||
|
||||
function escapeXml(value: string) {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """);
|
||||
}
|
||||
function list(value = "") {
|
||||
return value
|
||||
.split(",")
|
||||
|
||||
@@ -82,6 +82,52 @@ button {
|
||||
button:hover {
|
||||
border-color: var(--accent);
|
||||
}
|
||||
.file-button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 0.6rem;
|
||||
padding: 0.55rem 0.7rem;
|
||||
color: CanvasText;
|
||||
cursor: pointer;
|
||||
}
|
||||
.file-button input {
|
||||
position: absolute;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
overflow: hidden;
|
||||
clip: rect(0 0 0 0);
|
||||
}
|
||||
.analysis-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 0.55rem;
|
||||
margin: 0.7rem 0;
|
||||
}
|
||||
.analysis-grid div {
|
||||
padding: 0.55rem;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 0.55rem;
|
||||
}
|
||||
.analysis-grid :where(dt, dd) {
|
||||
margin: 0;
|
||||
}
|
||||
.analysis-grid dt {
|
||||
color: var(--muted);
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
.analysis-grid dd {
|
||||
font-weight: 700;
|
||||
}
|
||||
.run-actions {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
progress {
|
||||
width: 100%;
|
||||
accent-color: var(--accent);
|
||||
}
|
||||
label {
|
||||
display: grid;
|
||||
gap: 0.25rem;
|
||||
@@ -199,3 +245,8 @@ dialog {
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
@media (max-width: 540px) {
|
||||
.analysis-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"schemaVersion": 1,
|
||||
"id": "de.add-ideas.flow-tools",
|
||||
"name": "Flow Tools",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"description": "Transform structured data locally.",
|
||||
"entry": "./",
|
||||
"icon": "./favicon.svg",
|
||||
@@ -16,11 +16,49 @@
|
||||
},
|
||||
"requirements": {
|
||||
"secureContext": false,
|
||||
"workers": false,
|
||||
"workers": true,
|
||||
"indexedDb": false,
|
||||
"crossOriginIsolated": false,
|
||||
"topLevelContext": false
|
||||
},
|
||||
"io": {
|
||||
"accepts": [
|
||||
{
|
||||
"mediaType": "application/json",
|
||||
"extensions": [".json"]
|
||||
},
|
||||
{
|
||||
"mediaType": "application/x-ndjson",
|
||||
"extensions": [".ndjson", ".jsonl"]
|
||||
},
|
||||
{
|
||||
"mediaType": "text/csv",
|
||||
"extensions": [".csv"]
|
||||
},
|
||||
{
|
||||
"mediaType": "application/xml",
|
||||
"extensions": [".xml"]
|
||||
}
|
||||
],
|
||||
"produces": [
|
||||
{
|
||||
"mediaType": "application/json",
|
||||
"extensions": [".json"]
|
||||
},
|
||||
{
|
||||
"mediaType": "text/csv",
|
||||
"extensions": [".csv"]
|
||||
},
|
||||
{
|
||||
"mediaType": "image/svg+xml",
|
||||
"extensions": [".svg"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"required": ["workers"],
|
||||
"optional": []
|
||||
},
|
||||
"privacy": {
|
||||
"processing": "local",
|
||||
"fileUploads": true,
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
export const APP_VERSION = "0.1.0";
|
||||
export const APP_VERSION = "0.2.0";
|
||||
|
||||
@@ -58,6 +58,6 @@ test("PWA theme, offline reload, and manifest", async ({
|
||||
const m = await request.get("/deep/nested/flow/toolbox-app.json");
|
||||
await expect(m.json()).resolves.toMatchObject({
|
||||
id: "de.add-ideas.flow-tools",
|
||||
version: "0.1.0",
|
||||
version: "0.2.0",
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
test("keeps the primary workspace inside a narrow viewport", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page.goto("/deep/nested/flow/");
|
||||
await expect(page.locator("main").first()).toBeVisible();
|
||||
await expect(
|
||||
page.locator("main .loading, main .workbench-loading"),
|
||||
).toHaveCount(0);
|
||||
|
||||
const widths = await page.evaluate(() => ({
|
||||
content: document.documentElement.scrollWidth,
|
||||
viewport: document.documentElement.clientWidth,
|
||||
}));
|
||||
expect(widths.viewport).toBeLessThanOrEqual(430);
|
||||
expect(widths.content).toBeLessThanOrEqual(widths.viewport + 1);
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
importRecipe,
|
||||
records,
|
||||
renderPipelineSvg,
|
||||
runPipeline,
|
||||
type Stage,
|
||||
} from "../../src/core/pipeline";
|
||||
@@ -55,5 +56,55 @@ describe("pipeline", () => {
|
||||
'{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"type":"javascript","config":{}}]}',
|
||||
),
|
||||
).toThrow(/Invalid recipe/iu);
|
||||
expect(() =>
|
||||
importRecipe(
|
||||
'{"schema":"de.add-ideas.flow-tools.recipe.v1","stages":[{"id":"same","type":"sort","enabled":true,"config":{}},{"id":"same","type":"sort","enabled":true,"config":{}}]}',
|
||||
),
|
||||
).toThrow(/safe unique id/iu);
|
||||
});
|
||||
it("deduplicates, explodes arrays and slices deterministically", () => {
|
||||
const result = runPipeline(
|
||||
records([
|
||||
{ id: 1, tags: ["a", "b"] },
|
||||
{ id: 1, tags: ["a", "b"] },
|
||||
{ id: 2, tags: ["c"] },
|
||||
]),
|
||||
[
|
||||
{
|
||||
id: "d",
|
||||
type: "deduplicate",
|
||||
enabled: true,
|
||||
config: { fields: "id" },
|
||||
},
|
||||
{
|
||||
id: "e",
|
||||
type: "explode",
|
||||
enabled: true,
|
||||
config: { field: "tags", target: "tag" },
|
||||
},
|
||||
{
|
||||
id: "s",
|
||||
type: "slice",
|
||||
enabled: true,
|
||||
config: { offset: "1", count: "2" },
|
||||
},
|
||||
],
|
||||
);
|
||||
expect(result.rows.map((row) => row.tag)).toEqual(["b", "c"]);
|
||||
expect(result.analysis).toMatchObject({
|
||||
inputRows: 3,
|
||||
outputRows: 2,
|
||||
duplicateRows: 1,
|
||||
});
|
||||
expect(result.snapshots.at(-1)?.losses).toContain(
|
||||
"1 rows fall outside the selected slice.",
|
||||
);
|
||||
});
|
||||
it("exports an inert pipeline SVG", () => {
|
||||
const svg = renderPipelineSvg([
|
||||
{ id: "x", type: "map", enabled: true, config: {} },
|
||||
]);
|
||||
expect(svg).toContain("Flow pipeline");
|
||||
expect(svg).not.toContain("<script");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user