Release flow Tools v0.1.0
This commit is contained in:
@@ -0,0 +1,534 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import {
|
||||
stableStringify,
|
||||
stringifyCsv,
|
||||
triggerBlobDownload,
|
||||
type JsonValue,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import { parseData, type DataFormat } from "../core/data";
|
||||
import {
|
||||
exportRecipe,
|
||||
importRecipe,
|
||||
records,
|
||||
runPipeline,
|
||||
type PipelineResult,
|
||||
type Stage,
|
||||
type StageType,
|
||||
} from "../core/pipeline";
|
||||
const SAMPLE =
|
||||
'[{"name":"Ada","team":"blue","score":91},{"name":"Grace","team":"blue","score":84},{"name":"Linus","team":"green","score":73}]',
|
||||
RIGHT = '[{"team":"blue","lead":"Mina"},{"team":"green","lead":"Sam"}]';
|
||||
const DEFAULT: Stage[] = [
|
||||
{
|
||||
id: "stage-1",
|
||||
type: "filter",
|
||||
enabled: true,
|
||||
config: { field: "score", operator: ">=", value: "80" },
|
||||
},
|
||||
{
|
||||
id: "stage-2",
|
||||
type: "sort",
|
||||
enabled: true,
|
||||
config: { field: "score", direction: "desc" },
|
||||
},
|
||||
];
|
||||
export function Workbench() {
|
||||
const [source, setSource] = useState(SAMPLE),
|
||||
[format, setFormat] = useState<DataFormat>("json"),
|
||||
[input, setInput] = useState(() => records(parseData(SAMPLE, "json").rows)),
|
||||
[rightSource, setRightSource] = useState(RIGHT),
|
||||
[right, setRight] = useState(() => records(parseData(RIGHT, "json").rows)),
|
||||
[stages, setStages] = useState(DEFAULT),
|
||||
[result, setResult] = useState<PipelineResult>(() =>
|
||||
runPipeline(
|
||||
records(parseData(SAMPLE, "json").rows),
|
||||
DEFAULT,
|
||||
records(parseData(RIGHT, "json").rows),
|
||||
),
|
||||
),
|
||||
[selected, setSelected] = useState(2),
|
||||
[status, setStatus] = useState("Sample pipeline completed locally."),
|
||||
[recipeText, setRecipeText] = useState("");
|
||||
const snapshot =
|
||||
result.snapshots[Math.min(selected, result.snapshots.length - 1)]!;
|
||||
function parsePrimary() {
|
||||
try {
|
||||
const next = records(parseData(source, format).rows);
|
||||
setInput(next);
|
||||
setStatus(
|
||||
`Parsed ${next.length} primary rows. Run the recipe to replace results.`,
|
||||
);
|
||||
} catch (e) {
|
||||
setStatus(`${msg(e)} Last valid input retained.`);
|
||||
}
|
||||
}
|
||||
function parseRight() {
|
||||
try {
|
||||
const next = records(parseData(rightSource, "json").rows);
|
||||
setRight(next);
|
||||
setStatus(`Parsed ${next.length} secondary rows.`);
|
||||
} catch (e) {
|
||||
setStatus(`${msg(e)} Last valid secondary input retained.`);
|
||||
}
|
||||
}
|
||||
function run() {
|
||||
try {
|
||||
const next = runPipeline(input, stages, right);
|
||||
setResult(next);
|
||||
setSelected(next.snapshots.length - 1);
|
||||
setStatus(
|
||||
`Completed ${stages.filter((s) => s.enabled).length} stages; ${next.rows.length} rows remain.`,
|
||||
);
|
||||
} catch (e) {
|
||||
setStatus(`${msg(e)} Last successful stage snapshots remain visible.`);
|
||||
}
|
||||
}
|
||||
function update(index: number, stage: Stage) {
|
||||
setStages((s) => s.map((x, i) => (i === index ? stage : x)));
|
||||
}
|
||||
function add(type: StageType) {
|
||||
setStages((s) => [
|
||||
...s,
|
||||
{
|
||||
id: `stage-${Date.now().toString(36)}`,
|
||||
type,
|
||||
enabled: true,
|
||||
config: defaults(type),
|
||||
},
|
||||
]);
|
||||
}
|
||||
function download(kind: "json" | "csv" | "recipe") {
|
||||
if (kind === "recipe")
|
||||
triggerBlobDownload(
|
||||
new Blob([exportRecipe(stages)], { type: "application/json" }),
|
||||
"flow-recipe.json",
|
||||
);
|
||||
else if (kind === "json")
|
||||
triggerBlobDownload(
|
||||
new Blob([stableStringify(result.rows, 2)], {
|
||||
type: "application/json",
|
||||
}),
|
||||
"flow-result.json",
|
||||
);
|
||||
else {
|
||||
const fields = [...new Set(result.rows.flatMap(Object.keys))],
|
||||
table = [
|
||||
fields,
|
||||
...result.rows.map((row) =>
|
||||
fields.map((f) => scalar(row[f] ?? null)),
|
||||
),
|
||||
];
|
||||
triggerBlobDownload(
|
||||
new Blob([stringifyCsv(table)], { type: "text/csv" }),
|
||||
"flow-result.csv",
|
||||
);
|
||||
}
|
||||
}
|
||||
function loadRecipe() {
|
||||
try {
|
||||
const next = importRecipe(recipeText);
|
||||
setStages(next);
|
||||
setStatus(`Imported ${next.length} validated stages. Run to apply.`);
|
||||
} catch (e) {
|
||||
setStatus(msg(e));
|
||||
}
|
||||
}
|
||||
return (
|
||||
<main className="workbench">
|
||||
<section className="hero panel">
|
||||
<div>
|
||||
<p className="eyebrow">Inspectable local transformations</p>
|
||||
<h1>Build data flows without code.</h1>
|
||||
<p>
|
||||
Compose fixed primitives, inspect every transition and carry
|
||||
explicit losses with a deterministic recipe.
|
||||
</p>
|
||||
</div>
|
||||
<button className="primary" onClick={run}>
|
||||
Run complete pipeline
|
||||
</button>
|
||||
</section>
|
||||
<p role="status">{status}</p>
|
||||
<section className="sources">
|
||||
<section className="panel">
|
||||
<h2>Primary source</h2>
|
||||
<label>
|
||||
Format
|
||||
<select
|
||||
value={format}
|
||||
onChange={(e) => setFormat(e.target.value as DataFormat)}
|
||||
>
|
||||
<option value="json">JSON</option>
|
||||
<option value="csv">CSV</option>
|
||||
<option value="ndjson">NDJSON</option>
|
||||
<option value="xml">XML</option>
|
||||
</select>
|
||||
</label>
|
||||
<textarea
|
||||
value={source}
|
||||
onChange={(e) => setSource(e.target.value)}
|
||||
aria-label="Primary source"
|
||||
/>
|
||||
<button onClick={parsePrimary}>Parse primary</button>
|
||||
</section>
|
||||
<section className="panel">
|
||||
<h2>Secondary join source</h2>
|
||||
<p>JSON rows used only by Join stages.</p>
|
||||
<textarea
|
||||
value={rightSource}
|
||||
onChange={(e) => setRightSource(e.target.value)}
|
||||
aria-label="Secondary source"
|
||||
/>
|
||||
<button onClick={parseRight}>Parse secondary</button>
|
||||
</section>
|
||||
</section>
|
||||
<section className="pipeline">
|
||||
<div className="stage-list">
|
||||
<section className="panel add">
|
||||
<label>
|
||||
Add primitive
|
||||
<select id="add-stage" defaultValue="filter">
|
||||
<option value="select">Select fields</option>
|
||||
<option value="rename">Rename</option>
|
||||
<option value="filter">Filter</option>
|
||||
<option value="map">Map</option>
|
||||
<option value="sort">Sort</option>
|
||||
<option value="group">Group</option>
|
||||
<option value="join">Join</option>
|
||||
<option value="format">Format type</option>
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
onClick={() =>
|
||||
add(
|
||||
(document.getElementById("add-stage") as HTMLSelectElement)
|
||||
.value as StageType,
|
||||
)
|
||||
}
|
||||
>
|
||||
Add stage
|
||||
</button>
|
||||
</section>
|
||||
{stages.map((stage, index) => (
|
||||
<StageEditor
|
||||
key={stage.id}
|
||||
stage={stage}
|
||||
index={index}
|
||||
update={update}
|
||||
remove={() => setStages((s) => s.filter((_, i) => i !== index))}
|
||||
move={(delta) =>
|
||||
setStages((s) => {
|
||||
const target = index + delta;
|
||||
if (target < 0 || target >= s.length) return s;
|
||||
const n = [...s];
|
||||
[n[index], n[target]] = [n[target]!, n[index]!];
|
||||
return n;
|
||||
})
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<section className="panel inspector">
|
||||
<div className="snapshots">
|
||||
{result.snapshots.map((item, index) => (
|
||||
<button
|
||||
className={selected === index ? "active" : ""}
|
||||
key={`${item.name}-${index}`}
|
||||
onClick={() => setSelected(index)}
|
||||
>
|
||||
{index}. {item.name}
|
||||
<small>{item.rows.length} rows</small>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<h2>{snapshot.name}</h2>
|
||||
{snapshot.losses.length > 0 && (
|
||||
<ul className="losses">
|
||||
{snapshot.losses.slice(0, 50).map((x, i) => (
|
||||
<li key={i}>{x}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
<Table rows={snapshot.rows} />
|
||||
<div className="exports">
|
||||
<button onClick={() => download("json")}>Result JSON</button>
|
||||
<button onClick={() => download("csv")}>Result CSV</button>
|
||||
<button onClick={() => download("recipe")}>Recipe JSON</button>
|
||||
</div>
|
||||
<details>
|
||||
<summary>Import deterministic recipe</summary>
|
||||
<textarea
|
||||
value={recipeText}
|
||||
onChange={(e) => setRecipeText(e.target.value)}
|
||||
aria-label="Recipe JSON"
|
||||
/>
|
||||
<button onClick={loadRecipe}>Validate and import</button>
|
||||
</details>
|
||||
</section>
|
||||
</section>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
function StageEditor({
|
||||
stage,
|
||||
index,
|
||||
update,
|
||||
remove,
|
||||
move,
|
||||
}: {
|
||||
stage: Stage;
|
||||
index: number;
|
||||
update: (i: number, s: Stage) => void;
|
||||
remove: () => void;
|
||||
move: (d: number) => void;
|
||||
}) {
|
||||
const set = (key: string, value: string) =>
|
||||
update(index, { ...stage, config: { ...stage.config, [key]: value } });
|
||||
return (
|
||||
<article className="panel stage">
|
||||
<header>
|
||||
<strong>
|
||||
{index + 1}. {stage.type}
|
||||
</strong>
|
||||
<label>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={stage.enabled}
|
||||
onChange={(e) =>
|
||||
update(index, { ...stage, enabled: e.target.checked })
|
||||
}
|
||||
/>{" "}
|
||||
enabled
|
||||
</label>
|
||||
</header>
|
||||
<Config stage={stage} set={set} />
|
||||
<footer>
|
||||
<button onClick={() => move(-1)}>↑</button>
|
||||
<button onClick={() => move(1)}>↓</button>
|
||||
<button onClick={remove}>Remove</button>
|
||||
</footer>
|
||||
</article>
|
||||
);
|
||||
}
|
||||
function Config({
|
||||
stage,
|
||||
set,
|
||||
}: {
|
||||
stage: Stage;
|
||||
set: (k: string, v: string) => void;
|
||||
}) {
|
||||
const f = stage.config;
|
||||
if (stage.type === "select")
|
||||
return (
|
||||
<Field
|
||||
label="Fields (comma-separated)"
|
||||
value={f.fields}
|
||||
set={(v) => set("fields", v)}
|
||||
/>
|
||||
);
|
||||
if (stage.type === "rename")
|
||||
return (
|
||||
<Field
|
||||
label="Pairs old=new"
|
||||
value={f.pairs}
|
||||
set={(v) => set("pairs", v)}
|
||||
/>
|
||||
);
|
||||
if (stage.type === "filter")
|
||||
return (
|
||||
<>
|
||||
<Field label="Field" value={f.field} set={(v) => set("field", v)} />
|
||||
<Choice
|
||||
label="Operator"
|
||||
value={f.operator}
|
||||
values={["=", "!=", ">", ">=", "<", "<=", "contains"]}
|
||||
set={(v) => set("operator", v)}
|
||||
/>
|
||||
<Field label="Literal" value={f.value} set={(v) => set("value", v)} />
|
||||
</>
|
||||
);
|
||||
if (stage.type === "sort")
|
||||
return (
|
||||
<>
|
||||
<Field label="Field" value={f.field} set={(v) => set("field", v)} />
|
||||
<Choice
|
||||
label="Direction"
|
||||
value={f.direction}
|
||||
values={["asc", "desc"]}
|
||||
set={(v) => set("direction", v)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
if (stage.type === "group")
|
||||
return (
|
||||
<>
|
||||
<Field label="Group key" value={f.key} set={(v) => set("key", v)} />
|
||||
<Choice
|
||||
label="Aggregate"
|
||||
value={f.fn}
|
||||
values={["count", "sum", "avg", "min", "max"]}
|
||||
set={(v) => set("fn", v)}
|
||||
/>
|
||||
<Field
|
||||
label="Numeric field"
|
||||
value={f.field}
|
||||
set={(v) => set("field", v)}
|
||||
/>
|
||||
<Field
|
||||
label="Output field"
|
||||
value={f.output}
|
||||
set={(v) => set("output", v)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
if (stage.type === "join")
|
||||
return (
|
||||
<>
|
||||
<Field label="Left field" value={f.left} set={(v) => set("left", v)} />
|
||||
<Field
|
||||
label="Right field"
|
||||
value={f.right}
|
||||
set={(v) => set("right", v)}
|
||||
/>
|
||||
<Choice
|
||||
label="Mode"
|
||||
value={f.mode}
|
||||
values={["inner", "left"]}
|
||||
set={(v) => set("mode", v)}
|
||||
/>
|
||||
<Field
|
||||
label="Right prefix"
|
||||
value={f.prefix}
|
||||
set={(v) => set("prefix", v)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
if (stage.type === "format")
|
||||
return (
|
||||
<>
|
||||
<Field label="Field" value={f.field} set={(v) => set("field", v)} />
|
||||
<Choice
|
||||
label="As"
|
||||
value={f.as}
|
||||
values={["string", "number", "boolean"]}
|
||||
set={(v) => set("as", v)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
return (
|
||||
<>
|
||||
<Field label="Target" value={f.target} set={(v) => set("target", v)} />
|
||||
<Choice
|
||||
label="Operation"
|
||||
value={f.operation}
|
||||
values={["copy", "literal", "upper", "lower", "trim", "number"]}
|
||||
set={(v) => set("operation", v)}
|
||||
/>
|
||||
<Field
|
||||
label="Source field"
|
||||
value={f.source}
|
||||
set={(v) => set("source", v)}
|
||||
/>
|
||||
<Field label="Literal" value={f.value} set={(v) => set("value", v)} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
function Field({
|
||||
label,
|
||||
value = "",
|
||||
set,
|
||||
}: {
|
||||
label: string;
|
||||
value?: string;
|
||||
set: (v: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<label>
|
||||
{label}
|
||||
<input value={value} onChange={(e) => set(e.target.value)} />
|
||||
</label>
|
||||
);
|
||||
}
|
||||
function Choice({
|
||||
label,
|
||||
value = "",
|
||||
values,
|
||||
set,
|
||||
}: {
|
||||
label: string;
|
||||
value?: string;
|
||||
values: string[];
|
||||
set: (v: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<label>
|
||||
{label}
|
||||
<select value={value} onChange={(e) => set(e.target.value)}>
|
||||
{values.map((v) => (
|
||||
<option key={v}>{v}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
);
|
||||
}
|
||||
function Table({ rows }: { rows: Record<string, JsonValue>[] }) {
|
||||
const fields = useMemo(
|
||||
() => [...new Set(rows.flatMap(Object.keys))].slice(0, 100),
|
||||
[rows],
|
||||
);
|
||||
return (
|
||||
<div className="table-scroll">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
{fields.map((f) => (
|
||||
<th key={f}>{f}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.slice(0, 1000).map((row, i) => (
|
||||
<tr key={i}>
|
||||
{fields.map((f) => (
|
||||
<td key={f}>{scalar(row[f] ?? null)}</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{rows.length > 1000 && <p>Preview capped at 1,000 rows.</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
function scalar(v: JsonValue) {
|
||||
return typeof v === "object" && v !== null
|
||||
? stableStringify(v)
|
||||
: v === null
|
||||
? "null"
|
||||
: String(v);
|
||||
}
|
||||
function msg(e: unknown) {
|
||||
return e instanceof Error ? e.message : String(e);
|
||||
}
|
||||
function defaults(t: StageType): Record<string, string> {
|
||||
return t === "select"
|
||||
? { fields: "name, team" }
|
||||
: t === "rename"
|
||||
? { pairs: "name=full_name" }
|
||||
: t === "filter"
|
||||
? { field: "score", operator: ">=", value: "80" }
|
||||
: t === "map"
|
||||
? { target: "label", operation: "copy", source: "name", value: "" }
|
||||
: t === "sort"
|
||||
? { field: "score", direction: "desc" }
|
||||
: t === "group"
|
||||
? { key: "team", fn: "count", field: "score", output: "count" }
|
||||
: t === "join"
|
||||
? {
|
||||
left: "team",
|
||||
right: "team",
|
||||
mode: "left",
|
||||
prefix: "right_",
|
||||
}
|
||||
: { field: "score", as: "number" };
|
||||
}
|
||||
Reference in New Issue
Block a user