739 lines
21 KiB
TypeScript
739 lines
21 KiB
TypeScript
import { useMemo, useRef, useState } from "react";
|
|
import {
|
|
stableStringify,
|
|
stringifyCsv,
|
|
triggerBlobDownload,
|
|
type JsonValue,
|
|
} from "@add-ideas/toolbox-helpers";
|
|
import { parseData, type DataFormat } from "../core/data";
|
|
import { runFlowJob } from "../core/flow-client";
|
|
import {
|
|
exportRecipe,
|
|
importRecipe,
|
|
records,
|
|
renderPipelineSvg,
|
|
runPipeline,
|
|
type PipelineResult,
|
|
type Stage,
|
|
type StageType,
|
|
} from "../core/pipeline";
|
|
const SAMPLE =
|
|
'[{"name":"Ada","team":"blue","score":91},{"name":"Grace","team":"blue","score":84},{"name":"Linus","team":"green","score":73}]',
|
|
RIGHT = '[{"team":"blue","lead":"Mina"},{"team":"green","lead":"Sam"}]';
|
|
const DEFAULT: Stage[] = [
|
|
{
|
|
id: "stage-1",
|
|
type: "filter",
|
|
enabled: true,
|
|
config: { field: "score", operator: ">=", value: "80" },
|
|
},
|
|
{
|
|
id: "stage-2",
|
|
type: "sort",
|
|
enabled: true,
|
|
config: { field: "score", direction: "desc" },
|
|
},
|
|
];
|
|
export function Workbench() {
|
|
const [source, setSource] = useState(SAMPLE),
|
|
[format, setFormat] = useState<DataFormat>("json"),
|
|
[input, setInput] = useState(() => records(parseData(SAMPLE, "json").rows)),
|
|
[rightSource, setRightSource] = useState(RIGHT),
|
|
[right, setRight] = useState(() => records(parseData(RIGHT, "json").rows)),
|
|
[stages, setStages] = useState(DEFAULT),
|
|
[result, setResult] = useState<PipelineResult>(() =>
|
|
runPipeline(
|
|
records(parseData(SAMPLE, "json").rows),
|
|
DEFAULT,
|
|
records(parseData(RIGHT, "json").rows),
|
|
),
|
|
),
|
|
[selected, setSelected] = useState(2),
|
|
[status, setStatus] = useState("Sample pipeline completed locally."),
|
|
[recipeText, setRecipeText] = useState(""),
|
|
[busy, setBusy] = useState(false),
|
|
[progress, setProgress] = useState(0);
|
|
const operation = useRef<{ revision: number; controller?: AbortController }>({
|
|
revision: 0,
|
|
});
|
|
const snapshot =
|
|
result.snapshots[Math.min(selected, result.snapshots.length - 1)]!;
|
|
function parsePrimary() {
|
|
try {
|
|
const next = records(parseData(source, format).rows);
|
|
setInput(next);
|
|
setStatus(
|
|
`Parsed ${next.length} primary rows. Run the recipe to replace results.`,
|
|
);
|
|
} catch (e) {
|
|
setStatus(`${msg(e)} Last valid input retained.`);
|
|
}
|
|
}
|
|
function parseRight() {
|
|
try {
|
|
const next = records(parseData(rightSource, "json").rows);
|
|
setRight(next);
|
|
setStatus(`Parsed ${next.length} secondary rows.`);
|
|
} catch (e) {
|
|
setStatus(`${msg(e)} Last valid secondary input retained.`);
|
|
}
|
|
}
|
|
async function run() {
|
|
operation.current.controller?.abort();
|
|
const controller = new AbortController(),
|
|
revision = ++operation.current.revision;
|
|
operation.current.controller = controller;
|
|
setBusy(true);
|
|
setProgress(0);
|
|
try {
|
|
const next = await runFlowJob(input, stages, right, {
|
|
signal: controller.signal,
|
|
onProgress: (item) => {
|
|
if (operation.current.revision !== revision) return;
|
|
setProgress(item.total ? item.completed / item.total : 0);
|
|
setStatus(`Running ${item.stage} locally…`);
|
|
},
|
|
});
|
|
if (operation.current.revision !== revision) return;
|
|
setResult(next);
|
|
setSelected(next.snapshots.length - 1);
|
|
setStatus(
|
|
`Completed ${stages.filter((s) => s.enabled).length} stages; ${next.rows.length} rows remain.`,
|
|
);
|
|
} catch (e) {
|
|
if (operation.current.revision !== revision) return;
|
|
if (e instanceof DOMException && e.name === "AbortError") {
|
|
setStatus(
|
|
"Pipeline run cancelled. Last successful snapshots remain visible.",
|
|
);
|
|
return;
|
|
}
|
|
setStatus(`${msg(e)} Last successful stage snapshots remain visible.`);
|
|
} finally {
|
|
if (operation.current.revision === revision) {
|
|
setBusy(false);
|
|
operation.current.controller = undefined;
|
|
}
|
|
}
|
|
}
|
|
function update(index: number, stage: Stage) {
|
|
setStages((s) => s.map((x, i) => (i === index ? stage : x)));
|
|
}
|
|
function add(type: StageType) {
|
|
setStages((s) => [
|
|
...s,
|
|
{
|
|
id: nextStageId(s),
|
|
type,
|
|
enabled: true,
|
|
config: defaults(type),
|
|
},
|
|
]);
|
|
}
|
|
function download(kind: "json" | "csv" | "recipe" | "svg") {
|
|
if (kind === "recipe")
|
|
triggerBlobDownload(
|
|
new Blob([exportRecipe(stages)], { type: "application/json" }),
|
|
"flow-recipe.json",
|
|
);
|
|
else if (kind === "svg")
|
|
triggerBlobDownload(
|
|
new Blob([renderPipelineSvg(stages)], { type: "image/svg+xml" }),
|
|
"flow-pipeline.svg",
|
|
);
|
|
else if (kind === "json")
|
|
triggerBlobDownload(
|
|
new Blob([stableStringify(result.rows, 2)], {
|
|
type: "application/json",
|
|
}),
|
|
"flow-result.json",
|
|
);
|
|
else {
|
|
const fields = [...new Set(result.rows.flatMap(Object.keys))],
|
|
table = [
|
|
fields,
|
|
...result.rows.map((row) =>
|
|
fields.map((f) => scalar(row[f] ?? null)),
|
|
),
|
|
];
|
|
triggerBlobDownload(
|
|
new Blob([stringifyCsv(table)], { type: "text/csv" }),
|
|
"flow-result.csv",
|
|
);
|
|
}
|
|
}
|
|
async function openData(file: File | undefined, secondary = false) {
|
|
if (!file) return;
|
|
try {
|
|
if (file.size > 2 * 1024 * 1024)
|
|
throw new Error("Input file exceeds the 2 MiB limit.");
|
|
const text = await file.text(),
|
|
detected = inferFormat(file.name),
|
|
next = records(parseData(text, detected).rows);
|
|
if (secondary) {
|
|
setRightSource(text);
|
|
setRight(next);
|
|
setStatus(`Opened ${next.length} secondary rows from ${file.name}.`);
|
|
} else {
|
|
setFormat(detected);
|
|
setSource(text);
|
|
setInput(next);
|
|
setStatus(`Opened ${next.length} primary rows from ${file.name}.`);
|
|
}
|
|
} catch (e) {
|
|
setStatus(`${msg(e)} Last valid input retained.`);
|
|
}
|
|
}
|
|
async function openRecipe(file: File | undefined) {
|
|
if (!file) return;
|
|
try {
|
|
if (file.size > 256 * 1024) throw new Error("Recipe exceeds 256 KiB.");
|
|
const text = await file.text(),
|
|
next = importRecipe(text);
|
|
setRecipeText(text);
|
|
setStages(next);
|
|
setStatus(`Imported ${next.length} validated stages from ${file.name}.`);
|
|
} catch (e) {
|
|
setStatus(msg(e));
|
|
}
|
|
}
|
|
function loadRecipe() {
|
|
try {
|
|
const next = importRecipe(recipeText);
|
|
setStages(next);
|
|
setStatus(`Imported ${next.length} validated stages. Run to apply.`);
|
|
} catch (e) {
|
|
setStatus(msg(e));
|
|
}
|
|
}
|
|
return (
|
|
<main className="workbench">
|
|
<section className="hero panel">
|
|
<div>
|
|
<p className="eyebrow">Inspectable local transformations</p>
|
|
<h1>Build data flows without code.</h1>
|
|
<p>
|
|
Compose fixed primitives, inspect every transition and carry
|
|
explicit losses with a deterministic recipe.
|
|
</p>
|
|
</div>
|
|
<div className="run-actions">
|
|
<button
|
|
className="primary"
|
|
onClick={() => void run()}
|
|
disabled={busy}
|
|
>
|
|
Run complete pipeline
|
|
</button>
|
|
{busy && (
|
|
<button onClick={() => operation.current.controller?.abort()}>
|
|
Cancel
|
|
</button>
|
|
)}
|
|
</div>
|
|
</section>
|
|
<p role="status">{status}</p>
|
|
{busy && (
|
|
<progress value={progress} max={1} aria-label="Pipeline progress" />
|
|
)}
|
|
<section className="sources">
|
|
<section className="panel">
|
|
<h2>Primary source</h2>
|
|
<label>
|
|
Format
|
|
<select
|
|
value={format}
|
|
onChange={(e) => setFormat(e.target.value as DataFormat)}
|
|
>
|
|
<option value="json">JSON</option>
|
|
<option value="csv">CSV</option>
|
|
<option value="ndjson">NDJSON</option>
|
|
<option value="xml">XML</option>
|
|
</select>
|
|
</label>
|
|
<textarea
|
|
value={source}
|
|
onChange={(e) => setSource(e.target.value)}
|
|
aria-label="Primary source"
|
|
/>
|
|
<div className="exports">
|
|
<button onClick={parsePrimary}>Parse primary</button>
|
|
<label className="file-button">
|
|
Open data file
|
|
<input
|
|
type="file"
|
|
accept=".json,.csv,.ndjson,.jsonl,.xml"
|
|
onChange={(event) => void openData(event.target.files?.[0])}
|
|
/>
|
|
</label>
|
|
</div>
|
|
</section>
|
|
<section className="panel">
|
|
<h2>Secondary join source</h2>
|
|
<p>JSON rows used only by Join stages.</p>
|
|
<textarea
|
|
value={rightSource}
|
|
onChange={(e) => setRightSource(e.target.value)}
|
|
aria-label="Secondary source"
|
|
/>
|
|
<div className="exports">
|
|
<button onClick={parseRight}>Parse secondary</button>
|
|
<label className="file-button">
|
|
Open join file
|
|
<input
|
|
type="file"
|
|
accept=".json,.csv,.ndjson,.jsonl,.xml"
|
|
onChange={(event) =>
|
|
void openData(event.target.files?.[0], true)
|
|
}
|
|
/>
|
|
</label>
|
|
</div>
|
|
</section>
|
|
</section>
|
|
<section className="pipeline">
|
|
<div className="stage-list">
|
|
<section className="panel add">
|
|
<label>
|
|
Add primitive
|
|
<select id="add-stage" defaultValue="filter">
|
|
<option value="select">Select fields</option>
|
|
<option value="rename">Rename</option>
|
|
<option value="filter">Filter</option>
|
|
<option value="map">Map</option>
|
|
<option value="sort">Sort</option>
|
|
<option value="group">Group</option>
|
|
<option value="join">Join</option>
|
|
<option value="format">Format type</option>
|
|
<option value="deduplicate">Remove duplicates</option>
|
|
<option value="slice">Slice rows</option>
|
|
<option value="explode">Explode array</option>
|
|
</select>
|
|
</label>
|
|
<button
|
|
onClick={() =>
|
|
add(
|
|
(document.getElementById("add-stage") as HTMLSelectElement)
|
|
.value as StageType,
|
|
)
|
|
}
|
|
>
|
|
Add stage
|
|
</button>
|
|
</section>
|
|
{stages.map((stage, index) => (
|
|
<StageEditor
|
|
key={stage.id}
|
|
stage={stage}
|
|
index={index}
|
|
update={update}
|
|
remove={() => setStages((s) => s.filter((_, i) => i !== index))}
|
|
move={(delta) =>
|
|
setStages((s) => {
|
|
const target = index + delta;
|
|
if (target < 0 || target >= s.length) return s;
|
|
const n = [...s];
|
|
[n[index], n[target]] = [n[target]!, n[index]!];
|
|
return n;
|
|
})
|
|
}
|
|
/>
|
|
))}
|
|
</div>
|
|
<section className="panel inspector">
|
|
<div className="snapshots">
|
|
{result.snapshots.map((item, index) => (
|
|
<button
|
|
className={selected === index ? "active" : ""}
|
|
key={`${item.name}-${index}`}
|
|
onClick={() => setSelected(index)}
|
|
>
|
|
{index}. {item.name}
|
|
<small>{item.rows.length} rows</small>
|
|
</button>
|
|
))}
|
|
</div>
|
|
<h2>{snapshot.name}</h2>
|
|
{snapshot.losses.length > 0 && (
|
|
<ul className="losses">
|
|
{snapshot.losses.slice(0, 50).map((x, i) => (
|
|
<li key={i}>{x}</li>
|
|
))}
|
|
</ul>
|
|
)}
|
|
<Table rows={snapshot.rows} />
|
|
<dl className="analysis-grid" aria-label="Result analysis">
|
|
<div>
|
|
<dt>Rows</dt>
|
|
<dd>
|
|
{result.analysis.inputRows} → {result.analysis.outputRows}
|
|
</dd>
|
|
</div>
|
|
<div>
|
|
<dt>Fields</dt>
|
|
<dd>{result.analysis.fields}</dd>
|
|
</div>
|
|
<div>
|
|
<dt>Empty cells</dt>
|
|
<dd>{result.analysis.nullCells}</dd>
|
|
</div>
|
|
<div>
|
|
<dt>Input duplicates</dt>
|
|
<dd>{result.analysis.duplicateRows}</dd>
|
|
</div>
|
|
</dl>
|
|
<div className="exports">
|
|
<button onClick={() => download("json")}>Result JSON</button>
|
|
<button onClick={() => download("csv")}>Result CSV</button>
|
|
<button onClick={() => download("recipe")}>Recipe JSON</button>
|
|
<button onClick={() => download("svg")}>Pipeline SVG</button>
|
|
</div>
|
|
<details>
|
|
<summary>Import deterministic recipe</summary>
|
|
<textarea
|
|
value={recipeText}
|
|
onChange={(e) => setRecipeText(e.target.value)}
|
|
aria-label="Recipe JSON"
|
|
/>
|
|
<button onClick={loadRecipe}>Validate and import</button>
|
|
<label className="file-button">
|
|
Open recipe file
|
|
<input
|
|
type="file"
|
|
accept=".json,application/json"
|
|
onChange={(event) => void openRecipe(event.target.files?.[0])}
|
|
/>
|
|
</label>
|
|
</details>
|
|
</section>
|
|
</section>
|
|
</main>
|
|
);
|
|
}
|
|
function StageEditor({
|
|
stage,
|
|
index,
|
|
update,
|
|
remove,
|
|
move,
|
|
}: {
|
|
stage: Stage;
|
|
index: number;
|
|
update: (i: number, s: Stage) => void;
|
|
remove: () => void;
|
|
move: (d: number) => void;
|
|
}) {
|
|
const set = (key: string, value: string) =>
|
|
update(index, { ...stage, config: { ...stage.config, [key]: value } });
|
|
return (
|
|
<article className="panel stage">
|
|
<header>
|
|
<strong>
|
|
{index + 1}. {stage.type}
|
|
</strong>
|
|
<label>
|
|
<input
|
|
type="checkbox"
|
|
checked={stage.enabled}
|
|
onChange={(e) =>
|
|
update(index, { ...stage, enabled: e.target.checked })
|
|
}
|
|
/>{" "}
|
|
enabled
|
|
</label>
|
|
</header>
|
|
<Config stage={stage} set={set} />
|
|
<footer>
|
|
<button
|
|
type="button"
|
|
aria-label="Move stage up"
|
|
onClick={() => move(-1)}
|
|
>
|
|
↑
|
|
</button>
|
|
<button
|
|
type="button"
|
|
aria-label="Move stage down"
|
|
onClick={() => move(1)}
|
|
>
|
|
↓
|
|
</button>
|
|
<button onClick={remove}>Remove</button>
|
|
</footer>
|
|
</article>
|
|
);
|
|
}
|
|
function Config({
|
|
stage,
|
|
set,
|
|
}: {
|
|
stage: Stage;
|
|
set: (k: string, v: string) => void;
|
|
}) {
|
|
const f = stage.config;
|
|
if (stage.type === "select")
|
|
return (
|
|
<Field
|
|
label="Fields (comma-separated)"
|
|
value={f.fields}
|
|
set={(v) => set("fields", v)}
|
|
/>
|
|
);
|
|
if (stage.type === "rename")
|
|
return (
|
|
<Field
|
|
label="Pairs old=new"
|
|
value={f.pairs}
|
|
set={(v) => set("pairs", v)}
|
|
/>
|
|
);
|
|
if (stage.type === "filter")
|
|
return (
|
|
<>
|
|
<Field label="Field" value={f.field} set={(v) => set("field", v)} />
|
|
<Choice
|
|
label="Operator"
|
|
value={f.operator}
|
|
values={["=", "!=", ">", ">=", "<", "<=", "contains"]}
|
|
set={(v) => set("operator", v)}
|
|
/>
|
|
<Field label="Literal" value={f.value} set={(v) => set("value", v)} />
|
|
</>
|
|
);
|
|
if (stage.type === "sort")
|
|
return (
|
|
<>
|
|
<Field label="Field" value={f.field} set={(v) => set("field", v)} />
|
|
<Choice
|
|
label="Direction"
|
|
value={f.direction}
|
|
values={["asc", "desc"]}
|
|
set={(v) => set("direction", v)}
|
|
/>
|
|
</>
|
|
);
|
|
if (stage.type === "group")
|
|
return (
|
|
<>
|
|
<Field label="Group key" value={f.key} set={(v) => set("key", v)} />
|
|
<Choice
|
|
label="Aggregate"
|
|
value={f.fn}
|
|
values={["count", "sum", "avg", "min", "max"]}
|
|
set={(v) => set("fn", v)}
|
|
/>
|
|
<Field
|
|
label="Numeric field"
|
|
value={f.field}
|
|
set={(v) => set("field", v)}
|
|
/>
|
|
<Field
|
|
label="Output field"
|
|
value={f.output}
|
|
set={(v) => set("output", v)}
|
|
/>
|
|
</>
|
|
);
|
|
if (stage.type === "join")
|
|
return (
|
|
<>
|
|
<Field label="Left field" value={f.left} set={(v) => set("left", v)} />
|
|
<Field
|
|
label="Right field"
|
|
value={f.right}
|
|
set={(v) => set("right", v)}
|
|
/>
|
|
<Choice
|
|
label="Mode"
|
|
value={f.mode}
|
|
values={["inner", "left"]}
|
|
set={(v) => set("mode", v)}
|
|
/>
|
|
<Field
|
|
label="Right prefix"
|
|
value={f.prefix}
|
|
set={(v) => set("prefix", v)}
|
|
/>
|
|
</>
|
|
);
|
|
if (stage.type === "format")
|
|
return (
|
|
<>
|
|
<Field label="Field" value={f.field} set={(v) => set("field", v)} />
|
|
<Choice
|
|
label="As"
|
|
value={f.as}
|
|
values={["string", "number", "boolean"]}
|
|
set={(v) => set("as", v)}
|
|
/>
|
|
</>
|
|
);
|
|
if (stage.type === "deduplicate")
|
|
return (
|
|
<Field
|
|
label="Identity fields (blank = whole row)"
|
|
value={f.fields}
|
|
set={(v) => set("fields", v)}
|
|
/>
|
|
);
|
|
if (stage.type === "slice")
|
|
return (
|
|
<>
|
|
<Field label="Offset" value={f.offset} set={(v) => set("offset", v)} />
|
|
<Field label="Count" value={f.count} set={(v) => set("count", v)} />
|
|
</>
|
|
);
|
|
if (stage.type === "explode")
|
|
return (
|
|
<>
|
|
<Field
|
|
label="Array field"
|
|
value={f.field}
|
|
set={(v) => set("field", v)}
|
|
/>
|
|
<Field
|
|
label="Output field"
|
|
value={f.target}
|
|
set={(v) => set("target", v)}
|
|
/>
|
|
</>
|
|
);
|
|
return (
|
|
<>
|
|
<Field label="Target" value={f.target} set={(v) => set("target", v)} />
|
|
<Choice
|
|
label="Operation"
|
|
value={f.operation}
|
|
values={["copy", "literal", "upper", "lower", "trim", "number"]}
|
|
set={(v) => set("operation", v)}
|
|
/>
|
|
<Field
|
|
label="Source field"
|
|
value={f.source}
|
|
set={(v) => set("source", v)}
|
|
/>
|
|
<Field label="Literal" value={f.value} set={(v) => set("value", v)} />
|
|
</>
|
|
);
|
|
}
|
|
function Field({
|
|
label,
|
|
value = "",
|
|
set,
|
|
}: {
|
|
label: string;
|
|
value?: string;
|
|
set: (v: string) => void;
|
|
}) {
|
|
return (
|
|
<label>
|
|
{label}
|
|
<input value={value} onChange={(e) => set(e.target.value)} />
|
|
</label>
|
|
);
|
|
}
|
|
function Choice({
|
|
label,
|
|
value = "",
|
|
values,
|
|
set,
|
|
}: {
|
|
label: string;
|
|
value?: string;
|
|
values: string[];
|
|
set: (v: string) => void;
|
|
}) {
|
|
return (
|
|
<label>
|
|
{label}
|
|
<select value={value} onChange={(e) => set(e.target.value)}>
|
|
{values.map((v) => (
|
|
<option key={v}>{v}</option>
|
|
))}
|
|
</select>
|
|
</label>
|
|
);
|
|
}
|
|
function Table({ rows }: { rows: Record<string, JsonValue>[] }) {
|
|
const fields = useMemo(
|
|
() => [...new Set(rows.flatMap(Object.keys))].slice(0, 100),
|
|
[rows],
|
|
);
|
|
return (
|
|
<div className="table-scroll">
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
{fields.map((f) => (
|
|
<th key={f}>{f}</th>
|
|
))}
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{rows.slice(0, 1000).map((row, i) => (
|
|
<tr key={i}>
|
|
{fields.map((f) => (
|
|
<td key={f}>{scalar(row[f] ?? null)}</td>
|
|
))}
|
|
</tr>
|
|
))}
|
|
</tbody>
|
|
</table>
|
|
{rows.length > 1000 && <p>Preview capped at 1,000 rows.</p>}
|
|
</div>
|
|
);
|
|
}
|
|
function scalar(v: JsonValue) {
|
|
return typeof v === "object" && v !== null
|
|
? stableStringify(v)
|
|
: v === null
|
|
? "null"
|
|
: String(v);
|
|
}
|
|
function msg(e: unknown) {
|
|
return e instanceof Error ? e.message : String(e);
|
|
}
|
|
function defaults(t: StageType): Record<string, string> {
|
|
return t === "select"
|
|
? { fields: "name, team" }
|
|
: t === "rename"
|
|
? { pairs: "name=full_name" }
|
|
: t === "filter"
|
|
? { field: "score", operator: ">=", value: "80" }
|
|
: t === "map"
|
|
? { target: "label", operation: "copy", source: "name", value: "" }
|
|
: t === "sort"
|
|
? { field: "score", direction: "desc" }
|
|
: t === "group"
|
|
? { key: "team", fn: "count", field: "score", output: "count" }
|
|
: t === "join"
|
|
? {
|
|
left: "team",
|
|
right: "team",
|
|
mode: "left",
|
|
prefix: "right_",
|
|
}
|
|
: t === "format"
|
|
? { field: "score", as: "number" }
|
|
: t === "deduplicate"
|
|
? { fields: "" }
|
|
: t === "slice"
|
|
? { offset: "0", count: "100" }
|
|
: { field: "items", target: "item" };
|
|
}
|
|
|
|
function inferFormat(fileName: string): DataFormat {
|
|
const extension = fileName.split(".").at(-1)?.toLowerCase();
|
|
if (extension === "csv") return "csv";
|
|
if (extension === "ndjson" || extension === "jsonl") return "ndjson";
|
|
if (extension === "xml") return "xml";
|
|
return "json";
|
|
}
|
|
|
|
function nextStageId(stages: readonly Stage[]): string {
|
|
const ids = new Set(stages.map((stage) => stage.id));
|
|
let suffix = 1;
|
|
while (ids.has(`stage-${suffix}`)) suffix += 1;
|
|
return `stage-${suffix}`;
|
|
}
|