Release Text Tools 0.2.0
This commit is contained in:
+204
-19
@@ -1,8 +1,8 @@
|
||||
import { useState } from "react";
|
||||
import {
|
||||
decodeText,
|
||||
encodeText,
|
||||
triggerBlobDownload,
|
||||
triggerBlobDownloads,
|
||||
type TextEncoding,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import {
|
||||
@@ -13,6 +13,11 @@ import {
|
||||
type StepType,
|
||||
type TransformStep,
|
||||
} from "../text/pipeline";
|
||||
import {
|
||||
createTextArtifactEvidence,
|
||||
decodeTextWithEvidence,
|
||||
type TextByteEvidence,
|
||||
} from "../text/evidence";
|
||||
|
||||
const initial = " Crème brûlée \r\nAlpha\nalpha\r\n Cedar \n";
|
||||
const initialSteps: TransformStep[] = [
|
||||
@@ -31,8 +36,16 @@ const STEP_LABELS: Record<StepType, string> = {
|
||||
normalize: "Unicode normalization",
|
||||
transliterate: "Best-effort transliteration",
|
||||
escape: "Escape / encode",
|
||||
unescape: "Decode / unescape strictly",
|
||||
wrap: "Wrap text",
|
||||
columns: "Select/reorder columns",
|
||||
"replace-literal": "Replace literal text",
|
||||
"prefix-lines": "Prefix lines",
|
||||
"suffix-lines": "Suffix lines",
|
||||
"filter-lines": "Keep matching lines",
|
||||
"number-lines": "Number lines",
|
||||
"join-lines": "Join lines",
|
||||
"reverse-lines": "Reverse line order",
|
||||
};
|
||||
|
||||
function Option({
|
||||
@@ -43,9 +56,13 @@ function Option({
|
||||
change: (option: string) => void;
|
||||
}) {
|
||||
if (
|
||||
["trim-lines", "trim-document", "dedupe-lines", "transliterate"].includes(
|
||||
step.type,
|
||||
)
|
||||
[
|
||||
"trim-lines",
|
||||
"trim-document",
|
||||
"dedupe-lines",
|
||||
"transliterate",
|
||||
"reverse-lines",
|
||||
].includes(step.type)
|
||||
)
|
||||
return <span className="muted">No options</span>;
|
||||
if (step.type === "line-endings")
|
||||
@@ -104,10 +121,10 @@ function Option({
|
||||
))}
|
||||
</select>
|
||||
);
|
||||
if (step.type === "escape")
|
||||
if (step.type === "escape" || step.type === "unescape")
|
||||
return (
|
||||
<select
|
||||
aria-label="Escape target"
|
||||
aria-label={step.type === "escape" ? "Escape target" : "Decode source"}
|
||||
value={step.option}
|
||||
onChange={(event) => change(event.target.value)}
|
||||
>
|
||||
@@ -136,13 +153,89 @@ function Option({
|
||||
onChange={(event) => change(event.target.value)}
|
||||
/>
|
||||
);
|
||||
if (step.type === "replace-literal") {
|
||||
let values: [string, string] = ["", ""];
|
||||
try {
|
||||
const parsed = JSON.parse(step.option) as unknown;
|
||||
if (
|
||||
Array.isArray(parsed) &&
|
||||
parsed.length === 2 &&
|
||||
parsed.every((value) => typeof value === "string")
|
||||
)
|
||||
values = parsed as [string, string];
|
||||
} catch {
|
||||
/* Keep editable empty values; apply will report invalid stored JSON. */
|
||||
}
|
||||
return (
|
||||
<div className="column-options">
|
||||
<input
|
||||
aria-label="Literal search text"
|
||||
value={values[0]}
|
||||
onChange={(event) =>
|
||||
change(JSON.stringify([event.target.value, values[1]]))
|
||||
}
|
||||
/>
|
||||
<input
|
||||
aria-label="Literal replacement text"
|
||||
value={values[1]}
|
||||
onChange={(event) =>
|
||||
change(JSON.stringify([values[0], event.target.value]))
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (
|
||||
[
|
||||
"prefix-lines",
|
||||
"suffix-lines",
|
||||
"filter-lines",
|
||||
"number-lines",
|
||||
"join-lines",
|
||||
].includes(step.type)
|
||||
)
|
||||
return (
|
||||
<input
|
||||
aria-label={`${STEP_LABELS[step.type]} option`}
|
||||
value={step.option}
|
||||
onChange={(event) => change(event.target.value)}
|
||||
placeholder={
|
||||
step.type === "join-lines"
|
||||
? "Delimiter; \\n and \\t supported"
|
||||
: "Text"
|
||||
}
|
||||
/>
|
||||
);
|
||||
const pieces = step.option.split("|");
|
||||
const mode = pieces.length >= 3 ? pieces[0]! : "literal";
|
||||
const delimiter = pieces.length >= 3 ? pieces[1]! : pieces[0] || ",";
|
||||
const order =
|
||||
pieces.length >= 3 ? pieces.slice(2).join("|") : pieces[1] || "1";
|
||||
const encode = (nextMode: string, nextDelimiter: string, nextOrder: string) =>
|
||||
change(`${nextMode}|${nextDelimiter}|${nextOrder}`);
|
||||
return (
|
||||
<input
|
||||
aria-label="Column settings"
|
||||
value={step.option}
|
||||
onChange={(event) => change(event.target.value)}
|
||||
placeholder=",|3,1,2"
|
||||
/>
|
||||
<div className="column-options">
|
||||
<select
|
||||
aria-label="Column parsing mode"
|
||||
value={mode}
|
||||
onChange={(event) => encode(event.target.value, delimiter, order)}
|
||||
>
|
||||
<option value="csv">Quoted CSV records</option>
|
||||
<option value="literal">Literal delimiter per line</option>
|
||||
</select>
|
||||
<input
|
||||
aria-label="Column delimiter"
|
||||
value={delimiter}
|
||||
onChange={(event) => encode(mode, event.target.value, order)}
|
||||
placeholder=", or \\t"
|
||||
/>
|
||||
<input
|
||||
aria-label="Column order"
|
||||
value={order}
|
||||
onChange={(event) => encode(mode, delimiter, event.target.value)}
|
||||
placeholder="3,1,2"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -188,6 +281,8 @@ export function Workbench() {
|
||||
const [fatalDecode, setFatalDecode] = useState(true);
|
||||
const [error, setError] = useState("");
|
||||
const [recipe, setRecipe] = useState("");
|
||||
const [sourceEvidence, setSourceEvidence] = useState<TextByteEvidence>();
|
||||
const [sourceName, setSourceName] = useState("pasted-text.txt");
|
||||
const outputLoss =
|
||||
outputEncoding === "latin1" &&
|
||||
[...result.output].some((character) => character.codePointAt(0)! > 255);
|
||||
@@ -218,16 +313,18 @@ export function Workbench() {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const decoded = decodeText(
|
||||
const decoded = decodeTextWithEvidence(
|
||||
new Uint8Array(await file.arrayBuffer()),
|
||||
inputEncoding,
|
||||
fatalDecode,
|
||||
);
|
||||
if (decoded.length > 2_000_000)
|
||||
if (decoded.text.length > 2_000_000)
|
||||
throw new Error(
|
||||
"Decoded text exceeds the 2,000,000 UTF-16-unit pipeline limit.",
|
||||
);
|
||||
setSource(decoded);
|
||||
setSource(decoded.text);
|
||||
setSourceEvidence(decoded.evidence);
|
||||
setSourceName(file.name);
|
||||
setError("");
|
||||
} catch (reason) {
|
||||
setError(
|
||||
@@ -253,7 +350,7 @@ export function Workbench() {
|
||||
};
|
||||
const exportRecipe = () => {
|
||||
const value = JSON.stringify(
|
||||
{ schemaVersion: 1, app: "text-tools", version: "0.1.0", steps },
|
||||
{ schemaVersion: 1, app: "text-tools", version: "0.2.0", steps },
|
||||
null,
|
||||
2,
|
||||
);
|
||||
@@ -263,6 +360,41 @@ export function Workbench() {
|
||||
"text-tools-recipe.json",
|
||||
);
|
||||
};
|
||||
const exportArtifact = async () => {
|
||||
try {
|
||||
const bytes = encodeText(result.output, outputEncoding);
|
||||
const evidence = await createTextArtifactEvidence({
|
||||
sourceName,
|
||||
sourceEvidence,
|
||||
sourceText: source,
|
||||
outputName: `transformed-${outputEncoding}.txt`,
|
||||
outputEncoding,
|
||||
outputBytes: bytes,
|
||||
pipeline: result,
|
||||
steps,
|
||||
});
|
||||
triggerBlobDownloads(
|
||||
[
|
||||
{
|
||||
blob: new Blob([bytes as BlobPart], { type: "text/plain" }),
|
||||
filename: evidence.output.name,
|
||||
},
|
||||
{
|
||||
blob: new Blob([JSON.stringify(evidence, null, 2) + "\n"], {
|
||||
type: "application/json",
|
||||
}),
|
||||
filename: "text-tools-artifact-evidence.json",
|
||||
},
|
||||
],
|
||||
{ maximumFiles: 2, order: "input", revokeDelayMs: 1_000 },
|
||||
);
|
||||
setError("");
|
||||
} catch (reason) {
|
||||
setError(
|
||||
reason instanceof Error ? reason.message : "Artifact export failed.",
|
||||
);
|
||||
}
|
||||
};
|
||||
const importRecipe = () => {
|
||||
try {
|
||||
if (recipe.length > 1_000_000)
|
||||
@@ -315,7 +447,8 @@ export function Workbench() {
|
||||
<h1>Text Tools</h1>
|
||||
<p>
|
||||
Build an ordered, visible transformation pipeline for normalization,
|
||||
lines, casing, escaping, wrapping, and columns.
|
||||
lines, casing, strict escaping/decoding, wrapping, and quoted or
|
||||
literal-delimited columns.
|
||||
</p>
|
||||
</div>
|
||||
<span className="privacy-pill">Browser-local</span>
|
||||
@@ -362,11 +495,54 @@ export function Workbench() {
|
||||
</div>
|
||||
<textarea
|
||||
value={source}
|
||||
onChange={(event) => setSource(event.target.value)}
|
||||
onChange={(event) => {
|
||||
setSource(event.target.value);
|
||||
setSourceEvidence(undefined);
|
||||
setSourceName("pasted-text.txt");
|
||||
}}
|
||||
spellCheck={false}
|
||||
aria-label="Text source"
|
||||
/>
|
||||
<Inventory value={source} />
|
||||
{sourceEvidence && (
|
||||
<details>
|
||||
<summary>Byte decoding evidence</summary>
|
||||
<dl className="inventory">
|
||||
<div>
|
||||
<dt>Selected / BOM</dt>
|
||||
<dd>
|
||||
{sourceEvidence.selectedEncoding} /{" "}
|
||||
{sourceEvidence.bom?.encoding ?? "none"}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>UTF-8 validation</dt>
|
||||
<dd>
|
||||
{sourceEvidence.utf8.valid
|
||||
? "valid"
|
||||
: `invalid at byte ${sourceEvidence.utf8.firstInvalidOffset}`}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Replacement characters</dt>
|
||||
<dd>{sourceEvidence.replacementCharacters}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Byte newlines CRLF / LF / CR</dt>
|
||||
<dd>
|
||||
{sourceEvidence.byteNewlines.crlf} /{" "}
|
||||
{sourceEvidence.byteNewlines.bareLf} /{" "}
|
||||
{sourceEvidence.byteNewlines.bareCr}
|
||||
</dd>
|
||||
</div>
|
||||
</dl>
|
||||
{sourceEvidence.warnings.map((warning) => (
|
||||
<p className="warning" key={warning}>
|
||||
{warning}
|
||||
</p>
|
||||
))}
|
||||
</details>
|
||||
)}
|
||||
</section>
|
||||
<section className="panel workspace" aria-labelledby="output-heading">
|
||||
<div className="panel-heading">
|
||||
@@ -386,6 +562,9 @@ export function Workbench() {
|
||||
<button type="button" onClick={download}>
|
||||
Download
|
||||
</button>
|
||||
<button type="button" onClick={() => void exportArtifact()}>
|
||||
Export artifact + evidence
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<textarea
|
||||
@@ -533,6 +712,7 @@ export function Workbench() {
|
||||
<summary>Versioned recipe import/export</summary>
|
||||
<div className="recipe">
|
||||
<textarea
|
||||
aria-label="Text Tools recipe JSON"
|
||||
value={recipe}
|
||||
onChange={(event) => setRecipe(event.target.value)}
|
||||
placeholder="Paste a Text Tools recipe JSON here."
|
||||
@@ -551,7 +731,12 @@ export function Workbench() {
|
||||
Encoding detection is limited to an explicit choice; no arbitrary
|
||||
charset guess is made. Transliteration, compatibility normalization,
|
||||
escaping, column omission, and narrow encodings can be lossy, so the
|
||||
exact source and output remain visible.
|
||||
exact source and output remain visible. Portable artifact evidence
|
||||
includes exact hashes, selected encodings, newline facts and the
|
||||
ordered recipe. SDK 0.3.0 provides the shared transfer contract, but
|
||||
one-click Open With remains disabled until the Portal consumer rollout
|
||||
is coordinated. This version provides explicit portable downloads in
|
||||
the meantime.
|
||||
</p>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
@@ -298,6 +298,14 @@ textarea {
|
||||
display: flex;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
.column-options {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(8rem, 1fr) minmax(4rem, 0.55fr) minmax(
|
||||
7rem,
|
||||
1fr
|
||||
);
|
||||
gap: 0.4rem;
|
||||
}
|
||||
.step-actions button {
|
||||
min-width: 2.55rem;
|
||||
padding: 0.4rem;
|
||||
@@ -361,6 +369,9 @@ summary {
|
||||
.steps li {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
.column-options {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@media (max-width: 42rem) {
|
||||
.hero {
|
||||
@@ -369,4 +380,7 @@ summary {
|
||||
.privacy-pill {
|
||||
order: -1;
|
||||
}
|
||||
.encoding-row {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
import {
|
||||
decodeText,
|
||||
digestHex,
|
||||
encodeText,
|
||||
type TextEncoding,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
|
||||
import {
|
||||
textInventory,
|
||||
type PipelineResult,
|
||||
type StepReport,
|
||||
type TransformStep,
|
||||
} from "./pipeline";
|
||||
|
||||
export interface TextByteEvidence {
|
||||
readonly schemaVersion: 1;
|
||||
readonly byteLength: number;
|
||||
readonly selectedEncoding: TextEncoding;
|
||||
readonly fatalDecode: boolean;
|
||||
readonly bom?: { encoding: Exclude<TextEncoding, "latin1">; bytes: number };
|
||||
readonly utf8: { valid: boolean; firstInvalidOffset?: number };
|
||||
readonly zeroBytes: {
|
||||
total: number;
|
||||
evenOffsets: number;
|
||||
oddOffsets: number;
|
||||
};
|
||||
readonly byteNewlines: { crlf: number; bareLf: number; bareCr: number };
|
||||
readonly replacementCharacters: number;
|
||||
readonly warnings: readonly string[];
|
||||
}
|
||||
|
||||
export interface TextArtifactEvidence {
|
||||
readonly schemaVersion: 1;
|
||||
readonly artifactType: "de.add-ideas.toolbox.text/v1";
|
||||
readonly createdBy: { app: "text-tools"; version: "0.2.0" };
|
||||
readonly source: {
|
||||
name: string;
|
||||
canonicalUtf8Sha256: string;
|
||||
inventory: ReturnType<typeof textInventory>;
|
||||
byteEvidence?: TextByteEvidence;
|
||||
};
|
||||
readonly output: {
|
||||
name: string;
|
||||
mediaType: "text/plain";
|
||||
encoding: TextEncoding;
|
||||
bytes: number;
|
||||
sha256: string;
|
||||
inventory: ReturnType<typeof textInventory>;
|
||||
};
|
||||
readonly pipeline: {
|
||||
steps: readonly TransformStep[];
|
||||
reports: readonly StepReport[];
|
||||
warnings: readonly string[];
|
||||
};
|
||||
readonly handoff: {
|
||||
supportedByThisBuild: false;
|
||||
note: string;
|
||||
};
|
||||
}
|
||||
|
||||
function bom(bytes: Uint8Array): TextByteEvidence["bom"] {
|
||||
if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
|
||||
return { encoding: "utf-8", bytes: 3 };
|
||||
if (bytes[0] === 0xff && bytes[1] === 0xfe)
|
||||
return { encoding: "utf-16le", bytes: 2 };
|
||||
if (bytes[0] === 0xfe && bytes[1] === 0xff)
|
||||
return { encoding: "utf-16be", bytes: 2 };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function utf8InvalidOffset(bytes: Uint8Array): number | undefined {
|
||||
const continuation = (index: number) =>
|
||||
index < bytes.length && (bytes[index]! & 0xc0) === 0x80;
|
||||
for (let index = 0; index < bytes.length; index += 1) {
|
||||
const first = bytes[index]!;
|
||||
if (first <= 0x7f) continue;
|
||||
if (first >= 0xc2 && first <= 0xdf) {
|
||||
if (!continuation(index + 1)) return index;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (first >= 0xe0 && first <= 0xef) {
|
||||
const second = bytes[index + 1];
|
||||
if (
|
||||
second === undefined ||
|
||||
(first === 0xe0 && (second < 0xa0 || second > 0xbf)) ||
|
||||
(first === 0xed && (second < 0x80 || second > 0x9f)) ||
|
||||
(first !== 0xe0 &&
|
||||
first !== 0xed &&
|
||||
(second < 0x80 || second > 0xbf)) ||
|
||||
!continuation(index + 2)
|
||||
)
|
||||
return index;
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (first >= 0xf0 && first <= 0xf4) {
|
||||
const second = bytes[index + 1];
|
||||
if (
|
||||
second === undefined ||
|
||||
(first === 0xf0 && (second < 0x90 || second > 0xbf)) ||
|
||||
(first === 0xf4 && (second < 0x80 || second > 0x8f)) ||
|
||||
(first !== 0xf0 &&
|
||||
first !== 0xf4 &&
|
||||
(second < 0x80 || second > 0xbf)) ||
|
||||
!continuation(index + 2) ||
|
||||
!continuation(index + 3)
|
||||
)
|
||||
return index;
|
||||
index += 3;
|
||||
continue;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function inspectByteNewlines(bytes: Uint8Array) {
|
||||
let crlf = 0;
|
||||
let bareLf = 0;
|
||||
let bareCr = 0;
|
||||
for (let index = 0; index < bytes.length; index += 1) {
|
||||
if (bytes[index] === 0x0d && bytes[index + 1] === 0x0a) {
|
||||
crlf += 1;
|
||||
index += 1;
|
||||
} else if (bytes[index] === 0x0a) bareLf += 1;
|
||||
else if (bytes[index] === 0x0d) bareCr += 1;
|
||||
}
|
||||
return { crlf, bareLf, bareCr };
|
||||
}
|
||||
|
||||
export function decodeTextWithEvidence(
|
||||
bytes: Uint8Array,
|
||||
selectedEncoding: TextEncoding,
|
||||
fatalDecode: boolean,
|
||||
): { text: string; evidence: TextByteEvidence } {
|
||||
if (bytes.byteLength > 16 * 1024 * 1024)
|
||||
throw new RangeError("Text byte evidence is limited to 16 MiB.");
|
||||
const detectedBom = bom(bytes);
|
||||
const invalidOffset = utf8InvalidOffset(bytes);
|
||||
let zeroTotal = 0;
|
||||
let zeroEven = 0;
|
||||
let zeroOdd = 0;
|
||||
bytes.forEach((value, index) => {
|
||||
if (value !== 0) return;
|
||||
zeroTotal += 1;
|
||||
if (index % 2) zeroOdd += 1;
|
||||
else zeroEven += 1;
|
||||
});
|
||||
const text = decodeText(bytes, selectedEncoding, fatalDecode);
|
||||
const warnings: string[] = [];
|
||||
if (detectedBom && detectedBom.encoding !== selectedEncoding)
|
||||
warnings.push(
|
||||
`The ${detectedBom.encoding.toUpperCase()} BOM conflicts with the selected ${selectedEncoding.toUpperCase()} decoder.`,
|
||||
);
|
||||
if (!detectedBom)
|
||||
warnings.push(
|
||||
"No byte-order mark is present; the selected encoding is an explicit user choice, not a detection claim.",
|
||||
);
|
||||
if (selectedEncoding.startsWith("utf-16") && bytes.length % 2)
|
||||
warnings.push("UTF-16 input has an odd trailing byte.");
|
||||
if (selectedEncoding !== "utf-8" && invalidOffset === undefined)
|
||||
warnings.push(
|
||||
"The same bytes are also well-formed UTF-8; encoding intent cannot be inferred from validity alone.",
|
||||
);
|
||||
const replacementCharacters = [...text].filter(
|
||||
(character) => character === "\uFFFD",
|
||||
).length;
|
||||
if (replacementCharacters)
|
||||
warnings.push(
|
||||
`${replacementCharacters} replacement character(s) appear in decoded text; they may be source data or decoder substitutions.`,
|
||||
);
|
||||
return {
|
||||
text,
|
||||
evidence: Object.freeze({
|
||||
schemaVersion: 1,
|
||||
byteLength: bytes.byteLength,
|
||||
selectedEncoding,
|
||||
fatalDecode,
|
||||
...(detectedBom ? { bom: detectedBom } : {}),
|
||||
utf8: {
|
||||
valid: invalidOffset === undefined,
|
||||
...(invalidOffset === undefined
|
||||
? {}
|
||||
: { firstInvalidOffset: invalidOffset }),
|
||||
},
|
||||
zeroBytes: {
|
||||
total: zeroTotal,
|
||||
evenOffsets: zeroEven,
|
||||
oddOffsets: zeroOdd,
|
||||
},
|
||||
byteNewlines: inspectByteNewlines(bytes),
|
||||
replacementCharacters,
|
||||
warnings: Object.freeze(warnings),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function createTextArtifactEvidence(input: {
|
||||
sourceName: string;
|
||||
sourceEvidence?: TextByteEvidence;
|
||||
sourceText: string;
|
||||
outputName: string;
|
||||
outputEncoding: TextEncoding;
|
||||
outputBytes: Uint8Array;
|
||||
pipeline: PipelineResult;
|
||||
steps: readonly TransformStep[];
|
||||
}): Promise<TextArtifactEvidence> {
|
||||
if (input.outputBytes.byteLength > 32 * 1024 * 1024)
|
||||
throw new RangeError("Artifact output exceeds the 32 MiB evidence limit.");
|
||||
const canonicalSource = encodeText(input.sourceText, "utf-8");
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
artifactType: "de.add-ideas.toolbox.text/v1",
|
||||
createdBy: {
|
||||
app: "text-tools" as const,
|
||||
version: "0.2.0" as const,
|
||||
},
|
||||
source: {
|
||||
name: input.sourceName,
|
||||
canonicalUtf8Sha256: await digestHex(
|
||||
canonicalSource,
|
||||
"SHA-256",
|
||||
32 * 1024 * 1024,
|
||||
),
|
||||
inventory: textInventory(input.sourceText),
|
||||
...(input.sourceEvidence ? { byteEvidence: input.sourceEvidence } : {}),
|
||||
},
|
||||
output: {
|
||||
name: input.outputName,
|
||||
mediaType: "text/plain" as const,
|
||||
encoding: input.outputEncoding,
|
||||
bytes: input.outputBytes.byteLength,
|
||||
sha256: await digestHex(input.outputBytes, "SHA-256", 32 * 1024 * 1024),
|
||||
inventory: textInventory(input.pipeline.output),
|
||||
},
|
||||
pipeline: {
|
||||
steps: Object.freeze(
|
||||
input.steps.map((step) => Object.freeze({ ...step })),
|
||||
),
|
||||
reports: Object.freeze(
|
||||
input.pipeline.steps.map((report) => Object.freeze({ ...report })),
|
||||
),
|
||||
warnings: Object.freeze([...input.pipeline.warnings]),
|
||||
},
|
||||
handoff: {
|
||||
supportedByThisBuild: false as const,
|
||||
note: "The portable files are ready for explicit local transfer. SDK 0.3.0 provides the shared contract, but Open With remains disabled until the coordinated Portal consumer rollout.",
|
||||
},
|
||||
});
|
||||
}
|
||||
+247
-12
@@ -1,9 +1,14 @@
|
||||
import {
|
||||
base64ToBytes,
|
||||
bytesToBase64,
|
||||
bytesToHex,
|
||||
convertLineEndings,
|
||||
decodeText,
|
||||
encodeText,
|
||||
hexToBytes,
|
||||
normalizeUnicode,
|
||||
parseCsv,
|
||||
stringifyCsv,
|
||||
transformCase,
|
||||
type CaseTransform,
|
||||
type LineEnding,
|
||||
@@ -20,8 +25,16 @@ export type StepType =
|
||||
| "normalize"
|
||||
| "transliterate"
|
||||
| "escape"
|
||||
| "unescape"
|
||||
| "wrap"
|
||||
| "columns";
|
||||
| "columns"
|
||||
| "replace-literal"
|
||||
| "prefix-lines"
|
||||
| "suffix-lines"
|
||||
| "filter-lines"
|
||||
| "number-lines"
|
||||
| "join-lines"
|
||||
| "reverse-lines";
|
||||
export interface TransformStep {
|
||||
id: string;
|
||||
type: StepType;
|
||||
@@ -86,11 +99,27 @@ function wrapText(value: string, width: number): string {
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function transformColumns(value: string, option: string): string {
|
||||
const [delimiter = ",", order = "1"] = option.split("|");
|
||||
function transformColumns(
|
||||
value: string,
|
||||
option: string,
|
||||
): {
|
||||
value: string;
|
||||
warning?: string;
|
||||
} {
|
||||
const pieces = option.split("|");
|
||||
const mode = pieces.length >= 3 ? pieces[0] : "literal";
|
||||
const delimiterInput = pieces.length >= 3 ? pieces[1] : pieces[0];
|
||||
const order = pieces.length >= 3 ? pieces.slice(2).join("|") : pieces[1];
|
||||
const delimiter = delimiterInput === "\\t" ? "\t" : delimiterInput || ",";
|
||||
if (!delimiter || delimiter.length > 8)
|
||||
throw new Error("Column delimiter must contain 1–8 characters.");
|
||||
const indices = order.split(",").map((entry) => Number(entry.trim()) - 1);
|
||||
if (mode !== "literal" && mode !== "csv")
|
||||
throw new Error("Column parsing mode must be csv or literal.");
|
||||
if (mode === "csv" && delimiter.length !== 1)
|
||||
throw new Error("Quoted CSV mode requires a one-character delimiter.");
|
||||
const indices = (order || "1")
|
||||
.split(",")
|
||||
.map((entry) => Number(entry.trim()) - 1);
|
||||
if (
|
||||
!indices.length ||
|
||||
indices.some(
|
||||
@@ -98,12 +127,95 @@ function transformColumns(value: string, option: string): string {
|
||||
)
|
||||
)
|
||||
throw new Error("Column order uses 1-based indices such as 3,1,2.");
|
||||
return lines(value)
|
||||
.map((line) => {
|
||||
const cells = line.split(delimiter);
|
||||
return indices.map((index) => cells[index] ?? "").join(delimiter);
|
||||
})
|
||||
.join("\n");
|
||||
if (mode === "literal")
|
||||
return {
|
||||
value: lines(value)
|
||||
.map((line) => {
|
||||
const cells = line.split(delimiter);
|
||||
return indices.map((index) => cells[index] ?? "").join(delimiter);
|
||||
})
|
||||
.join("\n"),
|
||||
};
|
||||
const rows = parseCsv(value, {
|
||||
delimiter,
|
||||
maxRows: 200_000,
|
||||
maxColumns: 1_000,
|
||||
maxFieldChars: 2_000_000,
|
||||
});
|
||||
return {
|
||||
value: stringifyCsv(
|
||||
rows.map((row) => indices.map((index) => row[index] ?? "")),
|
||||
{
|
||||
delimiter,
|
||||
maxRows: 200_000,
|
||||
maxColumns: 1_000,
|
||||
maxFieldChars: 2_000_000,
|
||||
},
|
||||
),
|
||||
warning:
|
||||
"Quoted CSV rows were parsed across embedded delimiters/newlines and serialized canonically with CRLF row endings.",
|
||||
};
|
||||
}
|
||||
|
||||
function strictHtmlUnescape(value: string): string {
|
||||
const entities: Record<string, string> = {
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
""": '"',
|
||||
"'": "'",
|
||||
};
|
||||
let output = "";
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
if (value[index] !== "&") {
|
||||
output += value[index];
|
||||
continue;
|
||||
}
|
||||
const end = value.indexOf(";", index + 1);
|
||||
if (end < 0 || end - index > 6)
|
||||
throw new SyntaxError(
|
||||
`HTML entity at UTF-16 index ${index} is malformed.`,
|
||||
);
|
||||
const entity = value.slice(index, end + 1);
|
||||
const decoded = entities[entity];
|
||||
if (decoded === undefined)
|
||||
throw new SyntaxError(
|
||||
`HTML entity ${entity} is not emitted by the matching escape stage.`,
|
||||
);
|
||||
output += decoded;
|
||||
index = end;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function strictJsonUnescape(value: string): string {
|
||||
const parsed = JSON.parse(`"${value}"`) as unknown;
|
||||
if (typeof parsed !== "string")
|
||||
throw new SyntaxError("JSON escape input is invalid.");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function unescapeValue(value: string, option: string): string {
|
||||
if (option === "json") return strictJsonUnescape(value);
|
||||
if (option === "html") return strictHtmlUnescape(value);
|
||||
if (option === "url") return decodeURIComponent(value);
|
||||
if (option === "base64")
|
||||
return decodeText(
|
||||
base64ToBytes(value, { maxOutputBytes: MAX_OUTPUT }),
|
||||
"utf-8",
|
||||
true,
|
||||
);
|
||||
if (option === "hex")
|
||||
return decodeText(
|
||||
hexToBytes(value, {
|
||||
maxOutputBytes: MAX_OUTPUT,
|
||||
allowWhitespace: false,
|
||||
allowPrefix: false,
|
||||
}),
|
||||
"utf-8",
|
||||
true,
|
||||
);
|
||||
throw new Error("Unsupported unescape source.");
|
||||
}
|
||||
|
||||
function transliterate(value: string): string {
|
||||
@@ -119,6 +231,74 @@ function transliterate(value: string): string {
|
||||
.replaceAll("ł", "l");
|
||||
}
|
||||
|
||||
function parseReplacement(option: string): readonly [string, string] {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(option);
|
||||
} catch {
|
||||
throw new SyntaxError("Literal replacement options are invalid.");
|
||||
}
|
||||
if (
|
||||
!Array.isArray(parsed) ||
|
||||
parsed.length !== 2 ||
|
||||
parsed.some((value) => typeof value !== "string")
|
||||
)
|
||||
throw new SyntaxError(
|
||||
"Literal replacement requires [search, replacement].",
|
||||
);
|
||||
const [search, replacement] = parsed as [string, string];
|
||||
if (!search) throw new SyntaxError("Literal search text must not be empty.");
|
||||
if (search.length > 100_000 || replacement.length > 1_000_000)
|
||||
throw new RangeError(
|
||||
"Literal replacement option exceeds its safety limit.",
|
||||
);
|
||||
return [search, replacement];
|
||||
}
|
||||
|
||||
function decodedDelimiter(option: string): string {
|
||||
if (option.length > 1_000)
|
||||
throw new RangeError("Line join delimiter exceeds 1,000 characters.");
|
||||
return option.replaceAll("\\n", "\n").replaceAll("\\t", "\t");
|
||||
}
|
||||
|
||||
function assertProjectedOutput(units: number): void {
|
||||
if (!Number.isSafeInteger(units) || units > MAX_OUTPUT)
|
||||
throw new RangeError(
|
||||
"Transformation would exceed the 32 MiB output limit.",
|
||||
);
|
||||
}
|
||||
|
||||
function replaceLiteral(
|
||||
value: string,
|
||||
search: string,
|
||||
replacement: string,
|
||||
): string {
|
||||
let matches = 0;
|
||||
let offset = 0;
|
||||
while ((offset = value.indexOf(search, offset)) >= 0) {
|
||||
matches += 1;
|
||||
offset += search.length;
|
||||
assertProjectedOutput(
|
||||
value.length + matches * (replacement.length - search.length),
|
||||
);
|
||||
}
|
||||
return value.replaceAll(search, replacement);
|
||||
}
|
||||
|
||||
function decorateLines(
|
||||
value: string,
|
||||
addition: string,
|
||||
side: "prefix" | "suffix",
|
||||
): string {
|
||||
if (addition.length > 100_000)
|
||||
throw new RangeError("Line decoration exceeds 100,000 UTF-16 units.");
|
||||
const values = lines(value);
|
||||
assertProjectedOutput(value.length + values.length * addition.length);
|
||||
return values
|
||||
.map((line) => (side === "prefix" ? addition + line : line + addition))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function applyStep(
|
||||
value: string,
|
||||
step: TransformStep,
|
||||
@@ -188,10 +368,57 @@ function applyStep(
|
||||
return { value: bytesToHex(encodeText(value)) };
|
||||
throw new Error("Unsupported escape target.");
|
||||
}
|
||||
case "unescape":
|
||||
return {
|
||||
value: unescapeValue(value, step.option),
|
||||
warning:
|
||||
step.option === "base64" || step.option === "hex"
|
||||
? "Decoded bytes are required to be well-formed UTF-8 text; arbitrary binary is rejected."
|
||||
: undefined,
|
||||
};
|
||||
case "wrap":
|
||||
return { value: wrapText(value, Number(step.option)) };
|
||||
case "columns":
|
||||
return { value: transformColumns(value, step.option) };
|
||||
return transformColumns(value, step.option);
|
||||
case "replace-literal": {
|
||||
const [search, replacement] = parseReplacement(step.option);
|
||||
return { value: replaceLiteral(value, search, replacement) };
|
||||
}
|
||||
case "prefix-lines":
|
||||
return { value: decorateLines(value, step.option, "prefix") };
|
||||
case "suffix-lines":
|
||||
return { value: decorateLines(value, step.option, "suffix") };
|
||||
case "filter-lines": {
|
||||
if (!step.option || step.option.length > 100_000)
|
||||
throw new SyntaxError("Line filter text must not be empty.");
|
||||
return {
|
||||
value: lines(value)
|
||||
.filter((line) => line.includes(step.option))
|
||||
.join("\n"),
|
||||
warning: "Line filtering removes every line without the literal text.",
|
||||
};
|
||||
}
|
||||
case "number-lines": {
|
||||
const start = Number(step.option || "1");
|
||||
if (!Number.isSafeInteger(start) || Math.abs(start) > 1_000_000_000)
|
||||
throw new RangeError("Line-number start must be a bounded integer.");
|
||||
return {
|
||||
value: lines(value)
|
||||
.map((line, index) => `${start + index}. ${line}`)
|
||||
.join("\n"),
|
||||
};
|
||||
}
|
||||
case "join-lines": {
|
||||
const values = lines(value);
|
||||
const delimiter = decodedDelimiter(step.option);
|
||||
assertProjectedOutput(
|
||||
values.reduce((total, line) => total + line.length, 0) +
|
||||
Math.max(0, values.length - 1) * delimiter.length,
|
||||
);
|
||||
return { value: values.join(delimiter) };
|
||||
}
|
||||
case "reverse-lines":
|
||||
return { value: lines(value).reverse().join("\n") };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,8 +470,16 @@ export function createStep(type: StepType): TransformStep {
|
||||
normalize: "NFC",
|
||||
transliterate: "",
|
||||
escape: "json",
|
||||
unescape: "json",
|
||||
wrap: "80",
|
||||
columns: ",|1",
|
||||
columns: "csv|,|1",
|
||||
"replace-literal": JSON.stringify(["old", "new"]),
|
||||
"prefix-lines": "> ",
|
||||
"suffix-lines": "",
|
||||
"filter-lines": "text",
|
||||
"number-lines": "1",
|
||||
"join-lines": " ",
|
||||
"reverse-lines": "",
|
||||
};
|
||||
return {
|
||||
id: `step-${++nextStepId}`,
|
||||
|
||||
@@ -3,12 +3,21 @@
|
||||
"schemaVersion": 1,
|
||||
"id": "de.add-ideas.text-tools",
|
||||
"name": "Text Tools",
|
||||
"version": "0.1.0",
|
||||
"description": "Transform and inspect plain text locally in the browser.",
|
||||
"version": "0.2.0",
|
||||
"description": "Compose text transforms and export encoding evidence locally.",
|
||||
"entry": "./",
|
||||
"icon": "./favicon.svg",
|
||||
"categories": ["text", "developer", "productivity"],
|
||||
"tags": ["text", "unicode", "normalize", "sort", "escape", "encoding"],
|
||||
"tags": [
|
||||
"text",
|
||||
"unicode",
|
||||
"pipeline",
|
||||
"normalize",
|
||||
"sort",
|
||||
"escape",
|
||||
"encoding",
|
||||
"newline"
|
||||
],
|
||||
"integration": {
|
||||
"contextVersion": 1,
|
||||
"launchModes": ["navigate", "new-tab"],
|
||||
@@ -21,6 +30,28 @@
|
||||
"crossOriginIsolated": false,
|
||||
"topLevelContext": false
|
||||
},
|
||||
"io": {
|
||||
"accepts": [
|
||||
{
|
||||
"mediaType": "text/*",
|
||||
"extensions": [".txt", ".csv", ".md", ".log"],
|
||||
"label": "Bounded text files"
|
||||
}
|
||||
],
|
||||
"produces": [
|
||||
{
|
||||
"mediaType": "text/plain",
|
||||
"extensions": [".txt"],
|
||||
"label": "Transformed text"
|
||||
},
|
||||
{
|
||||
"mediaType": "application/json",
|
||||
"extensions": [".json"],
|
||||
"label": "Recipe and artifact evidence"
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": { "required": [], "optional": ["web-crypto"] },
|
||||
"privacy": {
|
||||
"processing": "local",
|
||||
"fileUploads": true,
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
export const APP_VERSION = "0.1.0";
|
||||
export const APP_VERSION = "0.2.0";
|
||||
|
||||
Reference in New Issue
Block a user