69 lines
2.0 KiB
TypeScript
69 lines
2.0 KiB
TypeScript
import { stableStringify } from "@add-ideas/toolbox-helpers";
|
|
|
|
export interface CorpusFile {
|
|
path: string;
|
|
mediaType: string;
|
|
purpose: string;
|
|
content: string;
|
|
}
|
|
|
|
export const FORMAT_LAB_CORPUS: readonly CorpusFile[] = [
|
|
{
|
|
path: "structured/typed.json",
|
|
mediaType: "application/json",
|
|
purpose: "Scalar types, null, nesting, Unicode and formula-like text.",
|
|
content:
|
|
'[{"id":9007199254740991,"active":true,"missing":null,"nested":{"score":1.25},"text":"Café 🚲","formula":"=2+2"}]\n',
|
|
},
|
|
{
|
|
path: "structured/text-cells.csv",
|
|
mediaType: "text/csv",
|
|
purpose:
|
|
"Leading zeros, empty cells, literal null and spreadsheet formula risk.",
|
|
content: 'id,empty,literal,formula\r\n001,,null,"=2+2"\r\n',
|
|
},
|
|
{
|
|
path: "structured/records.ndjson",
|
|
mediaType: "application/x-ndjson",
|
|
purpose: "Record framing and non-ASCII text.",
|
|
content: '{"id":1,"text":"naïve"}\n{"id":2,"text":"東京"}\n',
|
|
},
|
|
{
|
|
path: "structured/typed.xml",
|
|
mediaType: "application/xml",
|
|
purpose: "Typed lab XML dialect and escaped characters.",
|
|
content:
|
|
'<?xml version="1.0" encoding="UTF-8"?>\n<records><row><field name="id" type="number">1</field><field name="text" type="string">A & B</field></row></records>\n',
|
|
},
|
|
];
|
|
|
|
export function corpusManifest(): string {
|
|
return `${stableStringify(
|
|
{
|
|
schema: "https://git.add-ideas.de/lotobo/format-lab/schema/corpus-v1",
|
|
corpusVersion: 1,
|
|
generator: "de.add-ideas.format-lab",
|
|
deterministic: true,
|
|
files: FORMAT_LAB_CORPUS.map((file) => ({
|
|
path: file.path,
|
|
mediaType: file.mediaType,
|
|
purpose: file.purpose,
|
|
utf8Bytes: new TextEncoder().encode(file.content).length,
|
|
})),
|
|
},
|
|
2,
|
|
)}\n`;
|
|
}
|
|
|
|
export function exportCorpusFiles(): CorpusFile[] {
|
|
return [
|
|
...FORMAT_LAB_CORPUS.map((file) => ({ ...file })),
|
|
{
|
|
path: "corpus-manifest.json",
|
|
mediaType: "application/json",
|
|
purpose: "Machine-readable deterministic corpus inventory.",
|
|
content: corpusManifest(),
|
|
},
|
|
];
|
|
}
|