Release flow Tools v0.1.0
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
import {
|
||||
parseCsv,
|
||||
safeJsonParse,
|
||||
type JsonValue,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import { DOMParser, type Element as XmlElement } from "@xmldom/xmldom";
|
||||
|
||||
export type DataFormat = "json" | "csv" | "ndjson" | "xml";
|
||||
export interface DataDocument {
|
||||
format: DataFormat;
|
||||
root: JsonValue;
|
||||
rows: JsonValue[];
|
||||
fields: string[];
|
||||
}
|
||||
const MAX_CHARS = 2 * 1024 * 1024,
|
||||
MAX_ROWS = 10_000,
|
||||
MAX_FIELDS = 200;
|
||||
|
||||
export function parseData(source: string, format: DataFormat): DataDocument {
|
||||
if (source.length > MAX_CHARS)
|
||||
throw new Error("Input exceeds the 2 MiB limit.");
|
||||
let root: JsonValue;
|
||||
if (format === "json")
|
||||
root = safeJsonParse(source, {
|
||||
maxTextChars: MAX_CHARS,
|
||||
maxDepth: 32,
|
||||
maxNodes: 200_000,
|
||||
});
|
||||
else if (format === "ndjson") root = parseNdjson(source);
|
||||
else if (format === "csv") root = parseCsvData(source);
|
||||
else root = parseXml(source);
|
||||
const rows = normalizeRows(root);
|
||||
if (rows.length > MAX_ROWS)
|
||||
throw new Error("Dataset exceeds 10,000 query rows.");
|
||||
const fields = [
|
||||
...new Set(
|
||||
rows.flatMap((row) => (isObject(row) ? Object.keys(row) : ["value"])),
|
||||
),
|
||||
];
|
||||
if (fields.length > MAX_FIELDS)
|
||||
throw new Error("Dataset exposes more than 200 top-level fields.");
|
||||
return { format, root, rows, fields: fields.sort() };
|
||||
}
|
||||
|
||||
function parseCsvData(source: string): JsonValue {
|
||||
const table = parseCsv(source, {
|
||||
maxRows: MAX_ROWS + 1,
|
||||
maxColumns: MAX_FIELDS,
|
||||
maxFieldChars: 100_000,
|
||||
});
|
||||
if (!table.length) return [];
|
||||
const headers = table[0]!.map(
|
||||
(value, index) => value.trim() || `column_${index + 1}`,
|
||||
);
|
||||
if (new Set(headers).size !== headers.length)
|
||||
throw new Error("CSV headers must be unique.");
|
||||
return table
|
||||
.slice(1)
|
||||
.filter((row) => row.some(Boolean))
|
||||
.map((row) =>
|
||||
Object.fromEntries(
|
||||
headers.map((header, index) => [header, inferScalar(row[index] ?? "")]),
|
||||
),
|
||||
) as JsonValue;
|
||||
}
|
||||
|
||||
function parseNdjson(source: string): JsonValue {
|
||||
const lines = source.split(/\r?\n/u).filter((line) => line.trim());
|
||||
if (lines.length > MAX_ROWS)
|
||||
throw new Error("NDJSON exceeds 10,000 records.");
|
||||
return lines.map((line, index) => {
|
||||
try {
|
||||
return safeJsonParse(line, {
|
||||
maxTextChars: 200_000,
|
||||
maxDepth: 32,
|
||||
maxNodes: 20_000,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`NDJSON line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function parseXml(source: string): JsonValue {
|
||||
if (/<!\s*(?:DOCTYPE|ENTITY)\b/iu.test(source))
|
||||
throw new Error("XML DTD and entity declarations are not accepted.");
|
||||
const errors: string[] = [];
|
||||
const document = new DOMParser({
|
||||
onError(level, message) {
|
||||
if (level !== "warning") errors.push(message);
|
||||
},
|
||||
}).parseFromString(source, "application/xml");
|
||||
if (
|
||||
errors.length ||
|
||||
document.getElementsByTagName("parsererror").length ||
|
||||
!document.documentElement
|
||||
)
|
||||
throw new Error(`Malformed XML${errors[0] ? `: ${errors[0]}` : "."}`);
|
||||
let nodes = 0;
|
||||
const convert = (element: XmlElement, depth: number): JsonValue => {
|
||||
if (++nodes > 100_000 || depth > 32)
|
||||
throw new Error("XML structure exceeds node/depth limits.");
|
||||
const output: Record<string, JsonValue> = {};
|
||||
for (let index = 0; index < element.attributes.length; index += 1) {
|
||||
const attribute = element.attributes.item(index);
|
||||
if (attribute) output[`@${attribute.name}`] = attribute.value;
|
||||
}
|
||||
const children = [
|
||||
...Array.from({ length: element.childNodes.length }, (_, index) =>
|
||||
element.childNodes.item(index),
|
||||
),
|
||||
].filter((node): node is XmlElement =>
|
||||
Boolean(node && node.nodeType === 1),
|
||||
);
|
||||
for (const child of children) {
|
||||
const value = convert(child, depth + 1),
|
||||
key = child.nodeName;
|
||||
const existing = output[key];
|
||||
output[key] =
|
||||
existing === undefined
|
||||
? value
|
||||
: Array.isArray(existing)
|
||||
? [...existing, value]
|
||||
: [existing, value];
|
||||
}
|
||||
const text = [
|
||||
...Array.from({ length: element.childNodes.length }, (_, index) =>
|
||||
element.childNodes.item(index),
|
||||
),
|
||||
]
|
||||
.filter((node) => node?.nodeType === 3 || node?.nodeType === 4)
|
||||
.map((node) => node?.nodeValue ?? "")
|
||||
.join("")
|
||||
.trim();
|
||||
if (!Object.keys(output).length) return inferScalar(text);
|
||||
if (text) output["#text"] = text;
|
||||
return output;
|
||||
};
|
||||
return {
|
||||
[document.documentElement.nodeName]: convert(document.documentElement, 0),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeRows(value: JsonValue): JsonValue[] {
|
||||
if (Array.isArray(value)) return value;
|
||||
if (isObject(value)) {
|
||||
const arrays = Object.values(value).filter(Array.isArray);
|
||||
if (arrays.length === 1) return arrays[0] as JsonValue[];
|
||||
}
|
||||
return [value];
|
||||
}
|
||||
export function isObject(value: JsonValue): value is Record<string, JsonValue> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
function inferScalar(value: string): JsonValue {
|
||||
const trimmed = value.trim();
|
||||
if (/^-?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/iu.test(trimmed)) {
|
||||
const number = Number(trimmed);
|
||||
if (Number.isFinite(number)) return number;
|
||||
}
|
||||
if (/^(?:true|false)$/iu.test(trimmed))
|
||||
return trimmed.toLowerCase() === "true";
|
||||
if (/^null$/iu.test(trimmed)) return null;
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
import {
|
||||
safeJsonParse,
|
||||
stableStringify,
|
||||
type JsonValue,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import { isObject } from "./data";
|
||||
export type StageType =
|
||||
"select" | "rename" | "filter" | "map" | "sort" | "group" | "join" | "format";
|
||||
export interface Stage {
|
||||
id: string;
|
||||
type: StageType;
|
||||
enabled: boolean;
|
||||
config: Record<string, string>;
|
||||
}
|
||||
export interface Snapshot {
|
||||
name: string;
|
||||
rows: Record<string, JsonValue>[];
|
||||
losses: string[];
|
||||
stage?: Stage;
|
||||
}
|
||||
export interface PipelineResult {
|
||||
snapshots: Snapshot[];
|
||||
rows: Record<string, JsonValue>[];
|
||||
recipe: string;
|
||||
}
|
||||
const MAX_STAGES = 30,
|
||||
MAX_ROWS = 10000,
|
||||
MAX_JOIN = 20000;
|
||||
export function records(values: JsonValue[]): Record<string, JsonValue>[] {
|
||||
if (values.length > MAX_ROWS) throw new Error("Input exceeds 10,000 rows.");
|
||||
return values.map((value, index) =>
|
||||
isObject(value) ? { ...value } : { value, __row: index + 1 },
|
||||
);
|
||||
}
|
||||
export function runPipeline(
|
||||
input: Record<string, JsonValue>[],
|
||||
stages: Stage[],
|
||||
right: Record<string, JsonValue>[] = [],
|
||||
): PipelineResult {
|
||||
if (stages.length > MAX_STAGES) throw new Error("Recipe exceeds 30 stages.");
|
||||
let rows = input.map((row) => ({ ...row }));
|
||||
const snapshots: Snapshot[] = [{ name: "Input", rows, losses: [] }];
|
||||
for (const stage of stages) {
|
||||
if (!stage.enabled) {
|
||||
snapshots.push({
|
||||
name: `${stage.type} (disabled)`,
|
||||
rows,
|
||||
losses: [],
|
||||
stage,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const output = apply(rows, stage, right);
|
||||
rows = output.rows;
|
||||
if (rows.length > MAX_JOIN)
|
||||
throw new Error(`Stage ${stage.id} expands beyond 20,000 rows.`);
|
||||
snapshots.push({ name: stage.type, rows, losses: output.losses, stage });
|
||||
}
|
||||
return { snapshots, rows, recipe: exportRecipe(stages) };
|
||||
}
|
||||
function apply(
|
||||
rows: Record<string, JsonValue>[],
|
||||
stage: Stage,
|
||||
right: Record<string, JsonValue>[],
|
||||
) {
|
||||
const c = stage.config,
|
||||
losses: string[] = [];
|
||||
if (stage.type === "select") {
|
||||
const fields = list(c.fields);
|
||||
if (!fields.length) throw new Error("Select needs comma-separated fields.");
|
||||
return {
|
||||
rows: rows.map((row) =>
|
||||
Object.fromEntries(fields.map((field) => [field, row[field] ?? null])),
|
||||
),
|
||||
losses: [
|
||||
`Fields outside ${fields.join(", ")} were intentionally discarded.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
if (stage.type === "rename") {
|
||||
const pairs = list(c.pairs).map((item) =>
|
||||
item.split("=").map((x) => x.trim()),
|
||||
);
|
||||
return {
|
||||
rows: rows.map((row, index) => {
|
||||
const out = { ...row };
|
||||
for (const [left, target] of pairs) {
|
||||
if (!left || !target) throw new Error("Rename pairs use old=new.");
|
||||
if (target in out && target !== left)
|
||||
losses.push(
|
||||
`Row ${index + 1}: ${target} was overwritten by rename.`,
|
||||
);
|
||||
out[target] = out[left] ?? null;
|
||||
delete out[left];
|
||||
}
|
||||
return out;
|
||||
}),
|
||||
losses,
|
||||
};
|
||||
}
|
||||
if (stage.type === "filter") {
|
||||
const field = required(c.field, "Filter field"),
|
||||
op = c.operator ?? "=",
|
||||
literal = parseLiteral(c.value ?? "");
|
||||
return {
|
||||
rows: rows.filter((row) => test(row[field], op, literal)),
|
||||
losses: [
|
||||
`${rows.length} input rows were tested; non-matches were discarded.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
if (stage.type === "map") {
|
||||
const target = required(c.target, "Map target"),
|
||||
operation = c.operation ?? "copy",
|
||||
source = c.source ?? "",
|
||||
literal = c.value ?? "";
|
||||
return {
|
||||
rows: rows.map((row, index) => {
|
||||
const out = { ...row };
|
||||
let value: JsonValue =
|
||||
operation === "literal"
|
||||
? parseLiteral(literal)
|
||||
: (row[source] ?? null);
|
||||
if (
|
||||
operation === "upper" ||
|
||||
operation === "lower" ||
|
||||
operation === "trim"
|
||||
)
|
||||
value =
|
||||
operation === "upper"
|
||||
? String(value ?? "").toUpperCase()
|
||||
: operation === "lower"
|
||||
? String(value ?? "").toLowerCase()
|
||||
: String(value ?? "").trim();
|
||||
if (operation === "number") {
|
||||
const n = Number(value);
|
||||
if (!Number.isFinite(n)) {
|
||||
losses.push(
|
||||
`Row ${index + 1}: ${source} could not become a number; null emitted.`,
|
||||
);
|
||||
value = null;
|
||||
} else value = n;
|
||||
}
|
||||
out[target] = value;
|
||||
return out;
|
||||
}),
|
||||
losses,
|
||||
};
|
||||
}
|
||||
if (stage.type === "format") {
|
||||
const field = required(c.field, "Format field"),
|
||||
as = c.as ?? "string";
|
||||
return {
|
||||
rows: rows.map((row, index) => {
|
||||
const out = { ...row },
|
||||
value = row[field];
|
||||
if (as === "string")
|
||||
out[field] =
|
||||
value === null || value === undefined
|
||||
? ""
|
||||
: typeof value === "object"
|
||||
? stableStringify(value)
|
||||
: String(value);
|
||||
else if (as === "number") {
|
||||
const n = Number(value);
|
||||
out[field] = Number.isFinite(n) ? n : null;
|
||||
if (!Number.isFinite(n))
|
||||
losses.push(`Row ${index + 1}: ${field} became null.`);
|
||||
} else if (as === "boolean")
|
||||
out[field] = value === true || String(value).toLowerCase() === "true";
|
||||
else throw new Error("Format type must be string, number or boolean.");
|
||||
return out;
|
||||
}),
|
||||
losses,
|
||||
};
|
||||
}
|
||||
if (stage.type === "sort") {
|
||||
const field = required(c.field, "Sort field"),
|
||||
direction = c.direction === "desc" ? -1 : 1;
|
||||
return {
|
||||
rows: rows
|
||||
.map((row, index) => ({ row, index }))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
compare(a.row[field], b.row[field]) * direction ||
|
||||
a.index - b.index,
|
||||
)
|
||||
.map((item) => item.row),
|
||||
losses,
|
||||
};
|
||||
}
|
||||
if (stage.type === "group") {
|
||||
const key = required(c.key, "Group key"),
|
||||
fn = (c.fn ?? "count").toLowerCase(),
|
||||
field = c.field ?? "";
|
||||
const groups = new Map<
|
||||
string,
|
||||
{ value: JsonValue; rows: Record<string, JsonValue>[] }
|
||||
>();
|
||||
for (const row of rows) {
|
||||
const value = row[key] ?? null,
|
||||
id = stableStringify(value),
|
||||
group = groups.get(id) ?? { value, rows: [] };
|
||||
group.rows.push(row);
|
||||
groups.set(id, group);
|
||||
}
|
||||
return {
|
||||
rows: [...groups.values()].map((group) => {
|
||||
const values = group.rows
|
||||
.map((row) => row[field])
|
||||
.filter(
|
||||
(x): x is number => typeof x === "number" && Number.isFinite(x),
|
||||
);
|
||||
let aggregate: JsonValue;
|
||||
if (fn === "count") aggregate = group.rows.length;
|
||||
else if (!values.length) {
|
||||
aggregate = null;
|
||||
losses.push(
|
||||
`Group ${String(group.value)} has no numeric ${field} values.`,
|
||||
);
|
||||
} else if (fn === "sum") aggregate = values.reduce((a, b) => a + b, 0);
|
||||
else if (fn === "avg")
|
||||
aggregate = values.reduce((a, b) => a + b, 0) / values.length;
|
||||
else if (fn === "min") aggregate = Math.min(...values);
|
||||
else if (fn === "max") aggregate = Math.max(...values);
|
||||
else
|
||||
throw new Error(
|
||||
"Group function must be count, sum, avg, min or max.",
|
||||
);
|
||||
return { [key]: group.value, [c.output || fn]: aggregate };
|
||||
}),
|
||||
losses,
|
||||
};
|
||||
}
|
||||
if (stage.type === "join") {
|
||||
const leftKey = required(c.left, "Left join field"),
|
||||
rightKey = required(c.right, "Right join field"),
|
||||
mode = c.mode === "left" ? "left" : "inner",
|
||||
prefix = c.prefix ?? "right_",
|
||||
index = new Map<string, Record<string, JsonValue>[]>();
|
||||
for (const row of right) {
|
||||
const id = stableStringify(row[rightKey] ?? null),
|
||||
items = index.get(id) ?? [];
|
||||
items.push(row);
|
||||
index.set(id, items);
|
||||
}
|
||||
const joined: Record<string, JsonValue>[] = [];
|
||||
for (const row of rows) {
|
||||
const matches = index.get(stableStringify(row[leftKey] ?? null)) ?? [];
|
||||
if (!matches.length && mode === "left") joined.push({ ...row });
|
||||
for (const match of matches) {
|
||||
const out = { ...row };
|
||||
for (const [key, value] of Object.entries(match))
|
||||
out[`${prefix}${key}`] = value;
|
||||
joined.push(out);
|
||||
if (joined.length > MAX_JOIN)
|
||||
throw new Error("Join output exceeds 20,000 rows.");
|
||||
}
|
||||
}
|
||||
return {
|
||||
rows: joined,
|
||||
losses: [
|
||||
mode === "inner"
|
||||
? "Unmatched left rows were discarded."
|
||||
: "Unmatched left rows were retained; missing right fields are absent.",
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`Unsupported stage: ${stage.type}`);
|
||||
}
|
||||
export function exportRecipe(stages: Stage[]): string {
|
||||
return stableStringify(
|
||||
{ schema: "de.add-ideas.flow-tools.recipe.v1", stages },
|
||||
2,
|
||||
);
|
||||
}
|
||||
export function importRecipe(source: string): Stage[] {
|
||||
const value = safeJsonParse(source, {
|
||||
maxTextChars: 256 * 1024,
|
||||
maxDepth: 12,
|
||||
maxNodes: 10000,
|
||||
});
|
||||
if (
|
||||
!isObject(value) ||
|
||||
value.schema !== "de.add-ideas.flow-tools.recipe.v1" ||
|
||||
!Array.isArray(value.stages)
|
||||
)
|
||||
throw new Error("Not a Flow Tools recipe v1.");
|
||||
if (value.stages.length > MAX_STAGES)
|
||||
throw new Error("Recipe exceeds 30 stages.");
|
||||
return value.stages.map((raw, index) => {
|
||||
if (
|
||||
!isObject(raw) ||
|
||||
typeof raw.type !== "string" ||
|
||||
![
|
||||
"select",
|
||||
"rename",
|
||||
"filter",
|
||||
"map",
|
||||
"sort",
|
||||
"group",
|
||||
"join",
|
||||
"format",
|
||||
].includes(raw.type) ||
|
||||
!raw.config ||
|
||||
!isObject(raw.config)
|
||||
)
|
||||
throw new Error(`Invalid recipe stage ${index + 1}.`);
|
||||
const config: Record<string, string> = {};
|
||||
for (const [key, item] of Object.entries(raw.config))
|
||||
if (typeof item === "string") config[key] = item;
|
||||
else throw new Error(`Stage ${index + 1} config values must be strings.`);
|
||||
return {
|
||||
id: typeof raw.id === "string" ? raw.id : `stage-${index + 1}`,
|
||||
type: raw.type as StageType,
|
||||
enabled: raw.enabled !== false,
|
||||
config,
|
||||
};
|
||||
});
|
||||
}
|
||||
function list(value = "") {
|
||||
return value
|
||||
.split(",")
|
||||
.map((x) => x.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
function required(value: string | undefined, label: string) {
|
||||
if (!value?.trim()) throw new Error(`${label} is required.`);
|
||||
return value.trim();
|
||||
}
|
||||
function parseLiteral(value: string): JsonValue {
|
||||
const v = value.trim();
|
||||
if (
|
||||
(v.startsWith('"') && v.endsWith('"')) ||
|
||||
(v.startsWith("'") && v.endsWith("'"))
|
||||
)
|
||||
return v.slice(1, -1);
|
||||
if (/^-?(?:\d+\.?\d*|\.\d+)$/u.test(v)) return Number(v);
|
||||
if (/^(true|false)$/iu.test(v)) return v.toLowerCase() === "true";
|
||||
if (v === "null") return null;
|
||||
return value;
|
||||
}
|
||||
function test(left: JsonValue | undefined, op: string, right: JsonValue) {
|
||||
const c = compare(left, right);
|
||||
return op === "="
|
||||
? left === right
|
||||
: op === "!="
|
||||
? left !== right
|
||||
: op === ">"
|
||||
? c > 0
|
||||
: op === ">="
|
||||
? c >= 0
|
||||
: op === "<"
|
||||
? c < 0
|
||||
: op === "<="
|
||||
? c <= 0
|
||||
: op === "contains"
|
||||
? String(left ?? "").includes(String(right))
|
||||
: false;
|
||||
}
|
||||
function compare(a: JsonValue | undefined, b: JsonValue | undefined) {
|
||||
if (a === b) return 0;
|
||||
if (a == null) return -1;
|
||||
if (b == null) return 1;
|
||||
if (typeof a === "number" && typeof b === "number") return a - b;
|
||||
return String(a).localeCompare(String(b), undefined, { numeric: true });
|
||||
}
|
||||
Reference in New Issue
Block a user