Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 zemion
|
c40a2c4b2a |
@@ -0,0 +1,39 @@
|
||||
name: Verify
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: verify-${{ gitea.repository }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
CI: "true"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
- name: Select declared npm version
|
||||
run: npm install --global npm@11.17.0
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Audit runtime dependencies
|
||||
run: npm audit --omit=dev --audit-level=moderate
|
||||
- name: Check, test, and build
|
||||
run: npm run check
|
||||
- name: Install browser engines
|
||||
run: npx playwright install --with-deps chromium firefox webkit
|
||||
- name: Browser tests
|
||||
run: npm run test:browser
|
||||
@@ -1,5 +1,10 @@
|
||||
# Changelog
|
||||
|
||||
## 0.2.0 - 2026-09-02
|
||||
|
||||
- Add shared evidence envelopes, a reusable format catalogue and bounded local
|
||||
corpus runs for comparing measured round-trip behaviour across recipes.
|
||||
|
||||
## 0.1.0 - 2026-09-01
|
||||
|
||||
- Add property-aware conversion graphs for data, image, audio/video, and
|
||||
|
||||
@@ -4,7 +4,7 @@ Explore conversion paths, state the expected preservation boundary, and measure
|
||||
what a representative round trip actually changes. Format Lab is a local-first
|
||||
module in the add·ideas Toolbox.
|
||||
|
||||
## v0.1.0 capabilities
|
||||
## v0.2.0 capabilities
|
||||
|
||||
- Browse directed format graphs for structured data, images, audio/video, and
|
||||
subtitles
|
||||
@@ -20,11 +20,14 @@ module in the add·ideas Toolbox.
|
||||
untested when it is outside the canonical model
|
||||
- Export the target data and a deterministic evidence report containing the
|
||||
catalog version, exact path, expectations, measurements, and notices
|
||||
- Export a versioned portable evidence contract shared with other audit tools,
|
||||
inspect a broad masked-offset file-signature catalog, and download a
|
||||
deterministic edge-case corpus as individually named fixtures plus manifest
|
||||
|
||||
## Honest scope
|
||||
|
||||
The image, audio/video, and subtitle graphs are planning contracts linked to the
|
||||
specialized Toolbox apps; v0.1.0 does not duplicate their codec or renderer
|
||||
specialized Toolbox apps; v0.2.0 does not duplicate their codec or renderer
|
||||
engines. Catalog statuses are conservative expectations, not measurements of a
|
||||
particular file or encoder. The executable data lab supports records only—not
|
||||
arbitrary JSON documents, XML mixed content/namespaces, or an external CSV type
|
||||
@@ -34,6 +37,10 @@ Inputs are capped at 2 MiB, 10,000 records, 200 fields per record, 500,000
|
||||
top-level values, and 8 MiB output. CSV cells are strings and spreadsheet-formula
|
||||
prefixes are neutralized; the resulting semantic change is visible in the
|
||||
round-trip measurement.
|
||||
Signature matches identify byte patterns and likely containers only; they do
|
||||
not validate an entire file or establish that its content is safe. Parquet,
|
||||
Arrow, YAML, AVIF/HEIC/TIFF/GIF, Ogg/M4A/MOV/AAC and SBV/SAMI appear in the
|
||||
planning catalog without inventing executable conversion support.
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Corresponding source
|
||||
|
||||
The corresponding source for Format Lab 0.1.0 is available at:
|
||||
The corresponding source for Format Lab 0.2.0 is available at:
|
||||
|
||||
https://git.add-ideas.de/lotobo/format-lab/src/tag/v0.1.0
|
||||
https://git.add-ideas.de/lotobo/format-lab/src/tag/v0.2.0
|
||||
|
||||
Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`.
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# Third-party notices
|
||||
|
||||
Format Lab 0.1.0 directly depends on these runtime packages:
|
||||
Format Lab 0.2.0 directly depends on these runtime packages:
|
||||
|
||||
| Package | Pinned version | Declared licence |
|
||||
| -------------------------------- | -------------: | ---------------- |
|
||||
| `@add-ideas/toolbox-contract` | 0.2.3 | Apache-2.0 |
|
||||
| `@add-ideas/toolbox-helpers` | 0.1.0 | GPL-3.0-or-later |
|
||||
| `@add-ideas/toolbox-shell-react` | 0.2.3 | Apache-2.0 |
|
||||
| `@add-ideas/toolbox-contract` | 0.3.0 | Apache-2.0 |
|
||||
| `@add-ideas/toolbox-helpers` | 0.2.0 | GPL-3.0-or-later |
|
||||
| `@add-ideas/toolbox-shell-react` | 0.3.0 | Apache-2.0 |
|
||||
| `react` | 19.2.8 | MIT |
|
||||
| `react-dom` | 19.2.8 | MIT |
|
||||
|
||||
|
||||
Generated
+19
-19
@@ -1,22 +1,22 @@
|
||||
{
|
||||
"name": "format-lab",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "format-lab",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3",
|
||||
"@add-ideas/toolbox-helpers": "0.1.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.2.3",
|
||||
"@add-ideas/toolbox-contract": "0.3.0",
|
||||
"@add-ideas/toolbox-helpers": "0.2.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.3.0",
|
||||
"react": "19.2.8",
|
||||
"react-dom": "19.2.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@add-ideas/toolbox-testkit": "0.2.3",
|
||||
"@add-ideas/toolbox-testkit": "0.3.0",
|
||||
"@eslint/js": "10.0.1",
|
||||
"@playwright/test": "1.62.1",
|
||||
"@testing-library/jest-dom": "6.9.1",
|
||||
@@ -42,24 +42,25 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-contract": {
|
||||
"version": "0.2.3",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-contract/-/0.3.0/toolbox-contract-0.3.0.tgz",
|
||||
"integrity": "sha512-dKrK7BjOFwqJaBfJuhKxZKIld4sH0AKjEn6a0yLnbdMUFY+fFv4VSLGV2tNSBD016gumc2iNqOjUj/ld7x4rtA==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-helpers": {
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-shell-react": {
|
||||
"version": "0.2.3",
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-shell-react/-/0.3.0/toolbox-shell-react-0.3.0.tgz",
|
||||
"integrity": "sha512-74p6JzAOG0YCAKdlc1hLofV4ZIko7vb448S75cIiM88PKm93EHl5VD7g8YVyfM56Ui97UY9dmy+Whiq4sGzpsg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3"
|
||||
"@add-ideas/toolbox-contract": "0.3.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=18 <20",
|
||||
@@ -67,17 +68,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@add-ideas/toolbox-testkit": {
|
||||
"version": "0.2.3",
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://git.add-ideas.de/api/packages/lotobo/npm/%40add-ideas%2Ftoolbox-testkit/-/0.3.0/toolbox-testkit-0.3.0.tgz",
|
||||
"integrity": "sha512-4Fk+oSvZFspOMIXr8Xy040nhAaBsIQAzsGyXWSpjn3+k3yBKq7nB1r5zCHhsXzfdLzvPDAx2KcmSNOhM330D9w==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3"
|
||||
"@add-ideas/toolbox-contract": "0.3.0"
|
||||
},
|
||||
"bin": {
|
||||
"toolbox-check": "dist/cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/@adobe/css-tools": {
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "format-lab",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"description": "Explore conversion paths and measure format round-trip losses locally.",
|
||||
"license": "GPL-3.0-or-later",
|
||||
"author": "Albrecht Degering",
|
||||
@@ -39,14 +39,14 @@
|
||||
"release:artifact": "npm run check && npm run test:browser && npm run package:release -- --force"
|
||||
},
|
||||
"dependencies": {
|
||||
"@add-ideas/toolbox-contract": "0.2.3",
|
||||
"@add-ideas/toolbox-helpers": "0.1.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.2.3",
|
||||
"@add-ideas/toolbox-contract": "0.3.0",
|
||||
"@add-ideas/toolbox-helpers": "0.2.0",
|
||||
"@add-ideas/toolbox-shell-react": "0.3.0",
|
||||
"react": "19.2.8",
|
||||
"react-dom": "19.2.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@add-ideas/toolbox-testkit": "0.2.3",
|
||||
"@add-ideas/toolbox-testkit": "0.3.0",
|
||||
"@eslint/js": "10.0.1",
|
||||
"@playwright/test": "1.62.1",
|
||||
"@testing-library/jest-dom": "6.9.1",
|
||||
|
||||
+20
-2
@@ -15,7 +15,25 @@ export default defineConfig({
|
||||
timeout: 180_000,
|
||||
},
|
||||
projects: [
|
||||
{ name: "chromium", use: { ...devices["Desktop Chrome"] } },
|
||||
{ name: "firefox", use: { ...devices["Desktop Firefox"] } },
|
||||
{
|
||||
name: "chromium",
|
||||
testIgnore: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Desktop Chrome"] },
|
||||
},
|
||||
{
|
||||
name: "firefox",
|
||||
testIgnore: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Desktop Firefox"] },
|
||||
},
|
||||
{
|
||||
name: "webkit",
|
||||
testIgnore: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Desktop Safari"] },
|
||||
},
|
||||
{
|
||||
name: "mobile-chromium",
|
||||
testMatch: /responsive\.spec\.ts/,
|
||||
use: { ...devices["Pixel 5"] },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Changelog
|
||||
|
||||
## 0.2.0 - 2026-09-02
|
||||
|
||||
- Add shared evidence envelopes, a reusable format catalogue and bounded local
|
||||
corpus runs for comparing measured round-trip behaviour across recipes.
|
||||
|
||||
## 0.1.0 - 2026-09-01
|
||||
|
||||
- Add property-aware conversion graphs for data, image, audio/video, and
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
==============================================================================
|
||||
@add-ideas/toolbox-contract@0.2.3
|
||||
@add-ideas/toolbox-contract@0.3.0
|
||||
Declared licence: Apache-2.0
|
||||
==============================================================================
|
||||
--- LICENSE ---
|
||||
@@ -198,7 +198,7 @@ Declared licence: Apache-2.0
|
||||
|
||||
|
||||
==============================================================================
|
||||
@add-ideas/toolbox-helpers@0.1.0
|
||||
@add-ideas/toolbox-helpers@0.2.0
|
||||
Declared licence: GPL-3.0-or-later
|
||||
==============================================================================
|
||||
--- LICENSE ---
|
||||
@@ -879,7 +879,7 @@ Public License instead of this License. But first, please read
|
||||
|
||||
|
||||
==============================================================================
|
||||
@add-ideas/toolbox-shell-react@0.2.3
|
||||
@add-ideas/toolbox-shell-react@0.3.0
|
||||
Declared licence: Apache-2.0
|
||||
==============================================================================
|
||||
--- LICENSE ---
|
||||
|
||||
+9
-2
@@ -4,7 +4,7 @@ Explore conversion paths, state the expected preservation boundary, and measure
|
||||
what a representative round trip actually changes. Format Lab is a local-first
|
||||
module in the add·ideas Toolbox.
|
||||
|
||||
## v0.1.0 capabilities
|
||||
## v0.2.0 capabilities
|
||||
|
||||
- Browse directed format graphs for structured data, images, audio/video, and
|
||||
subtitles
|
||||
@@ -20,11 +20,14 @@ module in the add·ideas Toolbox.
|
||||
untested when it is outside the canonical model
|
||||
- Export the target data and a deterministic evidence report containing the
|
||||
catalog version, exact path, expectations, measurements, and notices
|
||||
- Export a versioned portable evidence contract shared with other audit tools,
|
||||
inspect a broad masked-offset file-signature catalog, and download a
|
||||
deterministic edge-case corpus as individually named fixtures plus manifest
|
||||
|
||||
## Honest scope
|
||||
|
||||
The image, audio/video, and subtitle graphs are planning contracts linked to the
|
||||
specialized Toolbox apps; v0.1.0 does not duplicate their codec or renderer
|
||||
specialized Toolbox apps; v0.2.0 does not duplicate their codec or renderer
|
||||
engines. Catalog statuses are conservative expectations, not measurements of a
|
||||
particular file or encoder. The executable data lab supports records only—not
|
||||
arbitrary JSON documents, XML mixed content/namespaces, or an external CSV type
|
||||
@@ -34,6 +37,10 @@ Inputs are capped at 2 MiB, 10,000 records, 200 fields per record, 500,000
|
||||
top-level values, and 8 MiB output. CSV cells are strings and spreadsheet-formula
|
||||
prefixes are neutralized; the resulting semantic change is visible in the
|
||||
round-trip measurement.
|
||||
Signature matches identify byte patterns and likely containers only; they do
|
||||
not validate an entire file or establish that its content is safe. Parquet,
|
||||
Arrow, YAML, AVIF/HEIC/TIFF/GIF, Ogg/M4A/MOV/AAC and SBV/SAMI appear in the
|
||||
planning catalog without inventing executable conversion support.
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
# Corresponding source
|
||||
|
||||
The corresponding source for Format Lab 0.1.0 is available at:
|
||||
The corresponding source for Format Lab 0.2.0 is available at:
|
||||
|
||||
https://git.add-ideas.de/lotobo/format-lab/src/tag/v0.1.0
|
||||
https://git.add-ideas.de/lotobo/format-lab/src/tag/v0.2.0
|
||||
|
||||
Build with Node.js 22, npm 11, `npm ci`, and `npm run release:artifact`.
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# Third-party notices
|
||||
|
||||
Format Lab 0.1.0 directly depends on these runtime packages:
|
||||
Format Lab 0.2.0 directly depends on these runtime packages:
|
||||
|
||||
| Package | Pinned version | Declared licence |
|
||||
| -------------------------------- | -------------: | ---------------- |
|
||||
| `@add-ideas/toolbox-contract` | 0.2.3 | Apache-2.0 |
|
||||
| `@add-ideas/toolbox-helpers` | 0.1.0 | GPL-3.0-or-later |
|
||||
| `@add-ideas/toolbox-shell-react` | 0.2.3 | Apache-2.0 |
|
||||
| `@add-ideas/toolbox-contract` | 0.3.0 | Apache-2.0 |
|
||||
| `@add-ideas/toolbox-helpers` | 0.2.0 | GPL-3.0-or-later |
|
||||
| `@add-ideas/toolbox-shell-react` | 0.3.0 | Apache-2.0 |
|
||||
| `react` | 19.2.8 | MIT |
|
||||
| `react-dom` | 19.2.8 | MIT |
|
||||
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
const CACHE_PREFIX = "format-lab-shell-";
|
||||
const CACHE_NAME = CACHE_PREFIX + "0.1.0";
|
||||
const CACHE_NAME = CACHE_PREFIX + "0.2.0";
|
||||
const CORE = ["./", "./manifest.webmanifest", "./favicon.svg"];
|
||||
self.addEventListener("install", (event) => {
|
||||
event.waitUntil(
|
||||
|
||||
+16
-1
@@ -3,7 +3,7 @@
|
||||
"schemaVersion": 1,
|
||||
"id": "de.add-ideas.format-lab",
|
||||
"name": "Format Lab",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"description": "Explore conversion paths and measure information loss locally.",
|
||||
"entry": "./",
|
||||
"icon": "./favicon.svg",
|
||||
@@ -28,6 +28,21 @@
|
||||
"crossOriginIsolated": false,
|
||||
"topLevelContext": false
|
||||
},
|
||||
"io": {
|
||||
"accepts": [
|
||||
{ "mediaType": "application/json", "extensions": [".json"] },
|
||||
{ "mediaType": "application/x-ndjson", "extensions": [".ndjson"] },
|
||||
{ "mediaType": "text/csv", "extensions": [".csv"] },
|
||||
{ "mediaType": "application/xml", "extensions": [".xml"] }
|
||||
],
|
||||
"produces": [
|
||||
{ "mediaType": "application/json", "extensions": [".json"] },
|
||||
{ "mediaType": "application/x-ndjson", "extensions": [".ndjson"] },
|
||||
{ "mediaType": "text/csv", "extensions": [".csv"] },
|
||||
{ "mediaType": "application/xml", "extensions": [".xml"] }
|
||||
]
|
||||
},
|
||||
"capabilities": { "required": [], "optional": [] },
|
||||
"privacy": {
|
||||
"processing": "local",
|
||||
"fileUploads": true,
|
||||
|
||||
+103
-13
@@ -1,7 +1,7 @@
|
||||
import { useMemo, useState, type ChangeEvent } from "react";
|
||||
import {
|
||||
stableStringify,
|
||||
triggerBlobDownload,
|
||||
triggerBlobDownloads,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import {
|
||||
domains,
|
||||
@@ -19,6 +19,13 @@ import {
|
||||
type DataFormat,
|
||||
type RoundTripResult,
|
||||
} from "../lab/data";
|
||||
import { exportCorpusFiles } from "../lab/corpus";
|
||||
import { serializeEvidence } from "../lab/evidence";
|
||||
import {
|
||||
formatSignatures,
|
||||
identifyBySignature,
|
||||
type FormatSignature,
|
||||
} from "../lab/signatures";
|
||||
|
||||
const SAMPLE = `[
|
||||
{
|
||||
@@ -159,6 +166,10 @@ export function Workbench() {
|
||||
"target",
|
||||
);
|
||||
const [error, setError] = useState<string>();
|
||||
const [signatureMatches, setSignatureMatches] = useState<FormatSignature[]>(
|
||||
[],
|
||||
);
|
||||
const [signatureName, setSignatureName] = useState("No file inspected");
|
||||
|
||||
const domainFormats = formats.filter((format) => format.domain === domain);
|
||||
const routes = useMemo(
|
||||
@@ -172,18 +183,7 @@ export function Workbench() {
|
||||
domainFormats.some((format) => format.id === edge.to),
|
||||
);
|
||||
const report = useMemo(
|
||||
() =>
|
||||
`${stableStringify(
|
||||
{
|
||||
catalogVersion: "format-catalog-v1",
|
||||
route: record.route.formats,
|
||||
expected: record.route.summary,
|
||||
measured: record.result.measurements,
|
||||
notes: record.result.notes,
|
||||
},
|
||||
2,
|
||||
{ maxTextChars: 2 * 1024 * 1024, maxDepth: 16, maxNodes: 20_000 },
|
||||
)}\n`,
|
||||
() => serializeEvidence(record.result.evidence),
|
||||
[record],
|
||||
);
|
||||
|
||||
@@ -240,6 +240,24 @@ export function Workbench() {
|
||||
setError(undefined);
|
||||
};
|
||||
|
||||
const inspectSignature = async (event: ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.target.files?.[0];
|
||||
event.target.value = "";
|
||||
if (!file) return;
|
||||
const bytes = new Uint8Array(await file.slice(0, 64).arrayBuffer());
|
||||
setSignatureName(file.name);
|
||||
setSignatureMatches(identifyBySignature(bytes));
|
||||
};
|
||||
|
||||
const downloadCorpus = () =>
|
||||
triggerBlobDownloads(
|
||||
exportCorpusFiles().map((file) => ({
|
||||
blob: new Blob([file.content], { type: file.mediaType }),
|
||||
filename: file.path.replaceAll("/", "__"),
|
||||
})),
|
||||
{ order: "filename", maximumFiles: 20 },
|
||||
);
|
||||
|
||||
const measuredByProperty = Object.fromEntries(
|
||||
record.result.measurements.map((measurement) => [
|
||||
measurement.property,
|
||||
@@ -408,6 +426,68 @@ export function Workbench() {
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<section className="panel" aria-labelledby="signature-title">
|
||||
<div className="panel-heading">
|
||||
<div>
|
||||
<h2 id="signature-title">
|
||||
Signature catalog and regression corpus
|
||||
</h2>
|
||||
<p>
|
||||
{formatSignatures.length} bounded byte signatures identify likely
|
||||
containers; they do not claim structural validity. {signatureName}
|
||||
.
|
||||
</p>
|
||||
</div>
|
||||
<div className="action-row">
|
||||
<label className="button file-button">
|
||||
Inspect first 64 bytes
|
||||
<input
|
||||
type="file"
|
||||
onChange={(event) => void inspectSignature(event)}
|
||||
/>
|
||||
</label>
|
||||
<button type="button" onClick={downloadCorpus}>
|
||||
Download fixture corpus
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<p className="notice-list">
|
||||
{signatureMatches.length
|
||||
? signatureMatches
|
||||
.map(
|
||||
(match) =>
|
||||
`${match.label} (${match.confidence}: ${match.note})`,
|
||||
)
|
||||
.join(" · ")
|
||||
: "No catalog signature matched the inspected prefix."}
|
||||
</p>
|
||||
<details>
|
||||
<summary>Browse all signature definitions</summary>
|
||||
<div className="table-scroll" tabIndex={0}>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
<th>Offset</th>
|
||||
<th>Confidence</th>
|
||||
<th>Boundary</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{formatSignatures.map((item) => (
|
||||
<tr key={item.id}>
|
||||
<td>{item.label}</td>
|
||||
<td>{item.offset}</td>
|
||||
<td>{item.confidence}</td>
|
||||
<td>{item.note}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<section className="panel" aria-labelledby="lab-title">
|
||||
<div className="panel-heading">
|
||||
<div>
|
||||
@@ -510,6 +590,16 @@ export function Workbench() {
|
||||
Download evidence report
|
||||
</button>
|
||||
</div>
|
||||
<details>
|
||||
<summary>Portable evidence JSON</summary>
|
||||
<textarea
|
||||
aria-label="Evidence report"
|
||||
readOnly
|
||||
value={report}
|
||||
rows={12}
|
||||
spellCheck={false}
|
||||
/>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -104,6 +104,24 @@ export const formats: FormatDefinition[] = [
|
||||
domain: "data",
|
||||
note: "Flexible tree with namespaces, attributes and mixed content.",
|
||||
},
|
||||
{
|
||||
id: "yaml",
|
||||
label: "YAML",
|
||||
domain: "data",
|
||||
note: "Typed graph-like syntax whose schema, tags, aliases and parser profile affect interoperability.",
|
||||
},
|
||||
{
|
||||
id: "parquet",
|
||||
label: "Parquet",
|
||||
domain: "data",
|
||||
note: "Columnar typed records with schemas, encodings and optional statistics.",
|
||||
},
|
||||
{
|
||||
id: "arrow",
|
||||
label: "Arrow IPC",
|
||||
domain: "data",
|
||||
note: "Typed columnar batches; extension metadata and dictionary handling matter.",
|
||||
},
|
||||
{
|
||||
id: "png",
|
||||
label: "PNG",
|
||||
@@ -128,6 +146,30 @@ export const formats: FormatDefinition[] = [
|
||||
domain: "image",
|
||||
note: "Vector/XML document with text and external-resource risks.",
|
||||
},
|
||||
{
|
||||
id: "gif",
|
||||
label: "GIF",
|
||||
domain: "image",
|
||||
note: "Palette raster with binary transparency and frame animation.",
|
||||
},
|
||||
{
|
||||
id: "avif",
|
||||
label: "AVIF",
|
||||
domain: "image",
|
||||
note: "AV1 image sequences in ISO BMFF with HDR/alpha profile variation.",
|
||||
},
|
||||
{
|
||||
id: "tiff",
|
||||
label: "TIFF",
|
||||
domain: "image",
|
||||
note: "Flexible tagged raster container with many compression and colour variants.",
|
||||
},
|
||||
{
|
||||
id: "heic",
|
||||
label: "HEIC",
|
||||
domain: "image",
|
||||
note: "HEVC image collections in ISO BMFF; browser decode support varies.",
|
||||
},
|
||||
{
|
||||
id: "wav",
|
||||
label: "WAV",
|
||||
@@ -170,6 +212,30 @@ export const formats: FormatDefinition[] = [
|
||||
domain: "av",
|
||||
note: "Flexible multi-track container with attachments and chapters.",
|
||||
},
|
||||
{
|
||||
id: "ogg",
|
||||
label: "Ogg",
|
||||
domain: "av",
|
||||
note: "Page-oriented container; actual preservation depends on codec streams and chaining.",
|
||||
},
|
||||
{
|
||||
id: "m4a",
|
||||
label: "M4A",
|
||||
domain: "av",
|
||||
note: "ISO BMFF audio convention, commonly AAC or ALAC with metadata.",
|
||||
},
|
||||
{
|
||||
id: "mov",
|
||||
label: "QuickTime MOV",
|
||||
domain: "av",
|
||||
note: "QuickTime/ISO BMFF family with broad codec and metadata options.",
|
||||
},
|
||||
{
|
||||
id: "aac",
|
||||
label: "AAC (ADTS)",
|
||||
domain: "av",
|
||||
note: "Raw framed lossy audio with limited document-level metadata.",
|
||||
},
|
||||
{
|
||||
id: "srt",
|
||||
label: "SRT",
|
||||
@@ -194,6 +260,18 @@ export const formats: FormatDefinition[] = [
|
||||
domain: "subtitle",
|
||||
note: "XML timing and styling profiles vary by ecosystem.",
|
||||
},
|
||||
{
|
||||
id: "sbv",
|
||||
label: "SBV",
|
||||
domain: "subtitle",
|
||||
note: "Simple timestamp pairs and text blocks with minimal styling.",
|
||||
},
|
||||
{
|
||||
id: "sami",
|
||||
label: "SAMI",
|
||||
domain: "subtitle",
|
||||
note: "HTML-like timed captions with CSS and legacy parser variation.",
|
||||
},
|
||||
];
|
||||
|
||||
function statuses(
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { stableStringify } from "@add-ideas/toolbox-helpers";
|
||||
|
||||
export interface CorpusFile {
|
||||
path: string;
|
||||
mediaType: string;
|
||||
purpose: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export const FORMAT_LAB_CORPUS: readonly CorpusFile[] = [
|
||||
{
|
||||
path: "structured/typed.json",
|
||||
mediaType: "application/json",
|
||||
purpose: "Scalar types, null, nesting, Unicode and formula-like text.",
|
||||
content:
|
||||
'[{"id":9007199254740991,"active":true,"missing":null,"nested":{"score":1.25},"text":"Café 🚲","formula":"=2+2"}]\n',
|
||||
},
|
||||
{
|
||||
path: "structured/text-cells.csv",
|
||||
mediaType: "text/csv",
|
||||
purpose:
|
||||
"Leading zeros, empty cells, literal null and spreadsheet formula risk.",
|
||||
content: 'id,empty,literal,formula\r\n001,,null,"=2+2"\r\n',
|
||||
},
|
||||
{
|
||||
path: "structured/records.ndjson",
|
||||
mediaType: "application/x-ndjson",
|
||||
purpose: "Record framing and non-ASCII text.",
|
||||
content: '{"id":1,"text":"naïve"}\n{"id":2,"text":"東京"}\n',
|
||||
},
|
||||
{
|
||||
path: "structured/typed.xml",
|
||||
mediaType: "application/xml",
|
||||
purpose: "Typed lab XML dialect and escaped characters.",
|
||||
content:
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n<records><row><field name="id" type="number">1</field><field name="text" type="string">A & B</field></row></records>\n',
|
||||
},
|
||||
];
|
||||
|
||||
export function corpusManifest(): string {
|
||||
return `${stableStringify(
|
||||
{
|
||||
schema: "https://git.add-ideas.de/lotobo/format-lab/schema/corpus-v1",
|
||||
corpusVersion: 1,
|
||||
generator: "de.add-ideas.format-lab",
|
||||
deterministic: true,
|
||||
files: FORMAT_LAB_CORPUS.map((file) => ({
|
||||
path: file.path,
|
||||
mediaType: file.mediaType,
|
||||
purpose: file.purpose,
|
||||
utf8Bytes: new TextEncoder().encode(file.content).length,
|
||||
})),
|
||||
},
|
||||
2,
|
||||
)}\n`;
|
||||
}
|
||||
|
||||
export function exportCorpusFiles(): CorpusFile[] {
|
||||
return [
|
||||
...FORMAT_LAB_CORPUS.map((file) => ({ ...file })),
|
||||
{
|
||||
path: "corpus-manifest.json",
|
||||
mediaType: "application/json",
|
||||
purpose: "Machine-readable deterministic corpus inventory.",
|
||||
content: corpusManifest(),
|
||||
},
|
||||
];
|
||||
}
|
||||
+37
-1
@@ -5,6 +5,7 @@ import {
|
||||
stringifyCsv,
|
||||
} from "@add-ideas/toolbox-helpers";
|
||||
import type { ConversionRoute } from "./catalog";
|
||||
import { createEvidenceReport, type EvidenceReport } from "./evidence";
|
||||
|
||||
export type DataFormat = "json" | "ndjson" | "csv" | "xml";
|
||||
export type MeasurementStatus =
|
||||
@@ -29,6 +30,7 @@ export interface RoundTripResult {
|
||||
roundTrip: DataDocument;
|
||||
measurements: PropertyMeasurement[];
|
||||
notes: string[];
|
||||
evidence: EvidenceReport;
|
||||
}
|
||||
|
||||
const MAX_SOURCE_BYTES = 2 * 1024 * 1024;
|
||||
@@ -445,13 +447,47 @@ export function runRoundTrip(
|
||||
(note) => `Round-trip ${sourceFormat.toUpperCase()}: ${note}`,
|
||||
),
|
||||
);
|
||||
const measurements = measureData(original, roundTrip);
|
||||
const evidence = createEvidenceReport({
|
||||
generator: { id: "de.add-ideas.format-lab", version: "0.2.0" },
|
||||
subject: {
|
||||
kind: "format-round-trip",
|
||||
name: route.formats.join(" → "),
|
||||
},
|
||||
operation: {
|
||||
id: "structured-round-trip",
|
||||
parameters: {
|
||||
route: [...route.formats],
|
||||
expected: { ...route.summary },
|
||||
},
|
||||
},
|
||||
limits: {
|
||||
maximumInputBytes: MAX_SOURCE_BYTES,
|
||||
maximumOutputBytes: MAX_OUTPUT_BYTES,
|
||||
maximumRows: MAX_ROWS,
|
||||
maximumColumns: MAX_COLUMNS,
|
||||
maximumValues: MAX_VALUES,
|
||||
},
|
||||
observations: measurements.map((measurement) => ({
|
||||
id: `property:${measurement.property}`,
|
||||
status:
|
||||
measurement.status === "measured-preserved"
|
||||
? "pass"
|
||||
: measurement.status === "measured-changed"
|
||||
? "change"
|
||||
: "not-tested",
|
||||
expectation: route.summary[measurement.property] ?? "unknown",
|
||||
detail: measurement.detail,
|
||||
})),
|
||||
});
|
||||
return {
|
||||
route: route.formats,
|
||||
targetText,
|
||||
roundTripText,
|
||||
original,
|
||||
roundTrip,
|
||||
measurements: measureData(original, roundTrip),
|
||||
measurements,
|
||||
notes: [...new Set(notes)],
|
||||
evidence,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { safeJsonParse, stableStringify } from "@add-ideas/toolbox-helpers";
|
||||
|
||||
export const EVIDENCE_SCHEMA =
|
||||
"https://git.add-ideas.de/lotobo/format-lab/schema/evidence-v1" as const;
|
||||
|
||||
export type EvidenceStatus = "pass" | "change" | "not-tested" | "informational";
|
||||
|
||||
export interface EvidenceObservation {
|
||||
id: string;
|
||||
status: EvidenceStatus;
|
||||
expectation?: string;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface EvidenceReport {
|
||||
schema: typeof EVIDENCE_SCHEMA;
|
||||
contractVersion: 1;
|
||||
generator: { id: string; version: string };
|
||||
subject: { kind: string; name: string; mediaType?: string };
|
||||
operation: { id: string; parameters: Record<string, unknown> };
|
||||
limits: Record<string, number | string | boolean>;
|
||||
observations: EvidenceObservation[];
|
||||
provenance: {
|
||||
execution: "local-browser";
|
||||
networkRequired: false;
|
||||
recordedAt: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
export function createEvidenceReport(
|
||||
input: Omit<EvidenceReport, "schema" | "contractVersion" | "provenance"> & {
|
||||
recordedAt?: string | null;
|
||||
},
|
||||
): EvidenceReport {
|
||||
if (!input.generator.id || !input.generator.version)
|
||||
throw new TypeError("Evidence generator identity is required.");
|
||||
if (!input.observations.length || input.observations.length > 1_000)
|
||||
throw new RangeError("Evidence reports need 1–1,000 observations.");
|
||||
const ids = new Set<string>();
|
||||
for (const observation of input.observations) {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/u.test(observation.id))
|
||||
throw new TypeError(`Invalid evidence observation id ${observation.id}.`);
|
||||
if (ids.has(observation.id))
|
||||
throw new TypeError(
|
||||
`Duplicate evidence observation id ${observation.id}.`,
|
||||
);
|
||||
ids.add(observation.id);
|
||||
}
|
||||
return {
|
||||
schema: EVIDENCE_SCHEMA,
|
||||
contractVersion: 1,
|
||||
generator: { ...input.generator },
|
||||
subject: { ...input.subject },
|
||||
operation: {
|
||||
id: input.operation.id,
|
||||
parameters: structuredClone(input.operation.parameters),
|
||||
},
|
||||
limits: { ...input.limits },
|
||||
observations: input.observations.map((item) => ({ ...item })),
|
||||
provenance: {
|
||||
execution: "local-browser",
|
||||
networkRequired: false,
|
||||
recordedAt: input.recordedAt ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeEvidence(report: EvidenceReport): string {
|
||||
return `${stableStringify(report, 2, {
|
||||
maxTextChars: 2 * 1024 * 1024,
|
||||
maxDepth: 24,
|
||||
maxNodes: 20_000,
|
||||
})}\n`;
|
||||
}
|
||||
|
||||
export function parseEvidence(source: string): EvidenceReport {
|
||||
const value = safeJsonParse(source, {
|
||||
maxTextChars: 2 * 1024 * 1024,
|
||||
maxDepth: 24,
|
||||
maxNodes: 20_000,
|
||||
rejectDangerousKeys: true,
|
||||
}) as unknown;
|
||||
if (!value || Array.isArray(value) || typeof value !== "object")
|
||||
throw new TypeError("Evidence must be a JSON object.");
|
||||
const candidate = value as Partial<EvidenceReport>;
|
||||
if (
|
||||
candidate.schema !== EVIDENCE_SCHEMA ||
|
||||
candidate.contractVersion !== 1 ||
|
||||
!candidate.generator ||
|
||||
!candidate.subject ||
|
||||
!candidate.operation ||
|
||||
!candidate.limits ||
|
||||
!Array.isArray(candidate.observations) ||
|
||||
!candidate.provenance ||
|
||||
candidate.provenance.execution !== "local-browser" ||
|
||||
candidate.provenance.networkRequired !== false
|
||||
)
|
||||
throw new TypeError("Unsupported or incomplete format evidence contract.");
|
||||
return createEvidenceReport({
|
||||
generator: candidate.generator,
|
||||
subject: candidate.subject,
|
||||
operation: candidate.operation,
|
||||
limits: candidate.limits,
|
||||
observations: candidate.observations,
|
||||
recordedAt: candidate.provenance.recordedAt,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
export interface FormatSignature {
|
||||
id: string;
|
||||
format: string;
|
||||
label: string;
|
||||
extensions: string[];
|
||||
mediaTypes: string[];
|
||||
offset: number;
|
||||
bytes: readonly number[];
|
||||
mask?: readonly number[];
|
||||
confidence: "strong" | "container" | "heuristic";
|
||||
note: string;
|
||||
}
|
||||
|
||||
const signature = (
|
||||
value: Omit<FormatSignature, "id"> & { id?: string },
|
||||
): FormatSignature => ({
|
||||
id: value.id ?? `${value.format}@${value.offset}`,
|
||||
...value,
|
||||
});
|
||||
|
||||
/** Byte signatures are identification hints, never a validation claim. */
|
||||
export const formatSignatures: readonly FormatSignature[] = [
|
||||
signature({
|
||||
format: "png",
|
||||
label: "PNG",
|
||||
extensions: ["png"],
|
||||
mediaTypes: ["image/png"],
|
||||
offset: 0,
|
||||
bytes: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a],
|
||||
confidence: "strong",
|
||||
note: "Complete PNG signature.",
|
||||
}),
|
||||
signature({
|
||||
format: "jpeg",
|
||||
label: "JPEG",
|
||||
extensions: ["jpg", "jpeg"],
|
||||
mediaTypes: ["image/jpeg"],
|
||||
offset: 0,
|
||||
bytes: [0xff, 0xd8, 0xff],
|
||||
confidence: "strong",
|
||||
note: "Start-of-image plus following marker prefix.",
|
||||
}),
|
||||
signature({
|
||||
format: "gif",
|
||||
label: "GIF87a",
|
||||
extensions: ["gif"],
|
||||
mediaTypes: ["image/gif"],
|
||||
offset: 0,
|
||||
bytes: [0x47, 0x49, 0x46, 0x38, 0x37, 0x61],
|
||||
confidence: "strong",
|
||||
note: "GIF87a header.",
|
||||
}),
|
||||
signature({
|
||||
id: "gif89a@0",
|
||||
format: "gif",
|
||||
label: "GIF89a",
|
||||
extensions: ["gif"],
|
||||
mediaTypes: ["image/gif"],
|
||||
offset: 0,
|
||||
bytes: [0x47, 0x49, 0x46, 0x38, 0x39, 0x61],
|
||||
confidence: "strong",
|
||||
note: "GIF89a header.",
|
||||
}),
|
||||
signature({
|
||||
format: "webp",
|
||||
label: "WebP",
|
||||
extensions: ["webp"],
|
||||
mediaTypes: ["image/webp"],
|
||||
offset: 0,
|
||||
bytes: [0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50],
|
||||
mask: [255, 255, 255, 255, 0, 0, 0, 0, 255, 255, 255, 255],
|
||||
confidence: "strong",
|
||||
note: "RIFF with WEBP form type; length bytes masked.",
|
||||
}),
|
||||
signature({
|
||||
format: "tiff-le",
|
||||
label: "TIFF little-endian",
|
||||
extensions: ["tif", "tiff"],
|
||||
mediaTypes: ["image/tiff"],
|
||||
offset: 0,
|
||||
bytes: [0x49, 0x49, 0x2a, 0x00],
|
||||
confidence: "strong",
|
||||
note: "Classic little-endian TIFF.",
|
||||
}),
|
||||
signature({
|
||||
format: "tiff-be",
|
||||
label: "TIFF big-endian",
|
||||
extensions: ["tif", "tiff"],
|
||||
mediaTypes: ["image/tiff"],
|
||||
offset: 0,
|
||||
bytes: [0x4d, 0x4d, 0x00, 0x2a],
|
||||
confidence: "strong",
|
||||
note: "Classic big-endian TIFF.",
|
||||
}),
|
||||
signature({
|
||||
format: "bmp",
|
||||
label: "BMP",
|
||||
extensions: ["bmp"],
|
||||
mediaTypes: ["image/bmp"],
|
||||
offset: 0,
|
||||
bytes: [0x42, 0x4d],
|
||||
confidence: "heuristic",
|
||||
note: "Bitmap file marker; structural validation still required.",
|
||||
}),
|
||||
signature({
|
||||
format: "pdf",
|
||||
label: "PDF",
|
||||
extensions: ["pdf"],
|
||||
mediaTypes: ["application/pdf"],
|
||||
offset: 0,
|
||||
bytes: [0x25, 0x50, 0x44, 0x46, 0x2d],
|
||||
confidence: "strong",
|
||||
note: "%PDF- header.",
|
||||
}),
|
||||
signature({
|
||||
format: "zip",
|
||||
label: "ZIP-compatible container",
|
||||
extensions: ["zip", "docx", "xlsx", "pptx", "epub", "odt", "ods", "odp"],
|
||||
mediaTypes: ["application/zip"],
|
||||
offset: 0,
|
||||
bytes: [0x50, 0x4b, 0x03, 0x04],
|
||||
confidence: "container",
|
||||
note: "Container only; entries distinguish ZIP-based formats.",
|
||||
}),
|
||||
signature({
|
||||
id: "zip-empty@0",
|
||||
format: "zip",
|
||||
label: "Empty ZIP",
|
||||
extensions: ["zip"],
|
||||
mediaTypes: ["application/zip"],
|
||||
offset: 0,
|
||||
bytes: [0x50, 0x4b, 0x05, 0x06],
|
||||
confidence: "container",
|
||||
note: "Empty archive end record.",
|
||||
}),
|
||||
signature({
|
||||
format: "gzip",
|
||||
label: "Gzip",
|
||||
extensions: ["gz", "tgz"],
|
||||
mediaTypes: ["application/gzip"],
|
||||
offset: 0,
|
||||
bytes: [0x1f, 0x8b, 0x08],
|
||||
confidence: "strong",
|
||||
note: "Gzip header with deflate method.",
|
||||
}),
|
||||
signature({
|
||||
format: "bzip2",
|
||||
label: "Bzip2",
|
||||
extensions: ["bz2"],
|
||||
mediaTypes: ["application/x-bzip2"],
|
||||
offset: 0,
|
||||
bytes: [0x42, 0x5a, 0x68],
|
||||
confidence: "strong",
|
||||
note: "BZip2 stream prefix.",
|
||||
}),
|
||||
signature({
|
||||
format: "xz",
|
||||
label: "XZ",
|
||||
extensions: ["xz"],
|
||||
mediaTypes: ["application/x-xz"],
|
||||
offset: 0,
|
||||
bytes: [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00],
|
||||
confidence: "strong",
|
||||
note: "XZ stream header.",
|
||||
}),
|
||||
signature({
|
||||
format: "7z",
|
||||
label: "7-Zip",
|
||||
extensions: ["7z"],
|
||||
mediaTypes: ["application/x-7z-compressed"],
|
||||
offset: 0,
|
||||
bytes: [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c],
|
||||
confidence: "strong",
|
||||
note: "7z signature.",
|
||||
}),
|
||||
signature({
|
||||
format: "rar4",
|
||||
label: "RAR 4",
|
||||
extensions: ["rar"],
|
||||
mediaTypes: ["application/vnd.rar"],
|
||||
offset: 0,
|
||||
bytes: [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00],
|
||||
confidence: "strong",
|
||||
note: "RAR 1.5–4.x signature.",
|
||||
}),
|
||||
signature({
|
||||
format: "rar5",
|
||||
label: "RAR 5",
|
||||
extensions: ["rar"],
|
||||
mediaTypes: ["application/vnd.rar"],
|
||||
offset: 0,
|
||||
bytes: [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00],
|
||||
confidence: "strong",
|
||||
note: "RAR 5 signature.",
|
||||
}),
|
||||
signature({
|
||||
format: "wasm",
|
||||
label: "WebAssembly",
|
||||
extensions: ["wasm"],
|
||||
mediaTypes: ["application/wasm"],
|
||||
offset: 0,
|
||||
bytes: [0x00, 0x61, 0x73, 0x6d],
|
||||
confidence: "strong",
|
||||
note: "WebAssembly magic; version follows.",
|
||||
}),
|
||||
signature({
|
||||
format: "elf",
|
||||
label: "ELF",
|
||||
extensions: [],
|
||||
mediaTypes: ["application/x-elf"],
|
||||
offset: 0,
|
||||
bytes: [0x7f, 0x45, 0x4c, 0x46],
|
||||
confidence: "strong",
|
||||
note: "ELF identification.",
|
||||
}),
|
||||
signature({
|
||||
format: "sqlite",
|
||||
label: "SQLite 3",
|
||||
extensions: ["sqlite", "db"],
|
||||
mediaTypes: ["application/vnd.sqlite3"],
|
||||
offset: 0,
|
||||
bytes: [
|
||||
0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61,
|
||||
0x74, 0x20, 0x33, 0x00,
|
||||
],
|
||||
confidence: "strong",
|
||||
note: "SQLite 3 database header.",
|
||||
}),
|
||||
signature({
|
||||
format: "flac",
|
||||
label: "FLAC",
|
||||
extensions: ["flac"],
|
||||
mediaTypes: ["audio/flac"],
|
||||
offset: 0,
|
||||
bytes: [0x66, 0x4c, 0x61, 0x43],
|
||||
confidence: "strong",
|
||||
note: "Native FLAC marker.",
|
||||
}),
|
||||
signature({
|
||||
format: "ogg",
|
||||
label: "Ogg",
|
||||
extensions: ["ogg", "oga", "ogv", "opus"],
|
||||
mediaTypes: ["application/ogg"],
|
||||
offset: 0,
|
||||
bytes: [0x4f, 0x67, 0x67, 0x53],
|
||||
confidence: "container",
|
||||
note: "Ogg page marker; codec requires packet inspection.",
|
||||
}),
|
||||
signature({
|
||||
format: "wav",
|
||||
label: "WAVE",
|
||||
extensions: ["wav"],
|
||||
mediaTypes: ["audio/wav"],
|
||||
offset: 0,
|
||||
bytes: [0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x41, 0x56, 0x45],
|
||||
mask: [255, 255, 255, 255, 0, 0, 0, 0, 255, 255, 255, 255],
|
||||
confidence: "container",
|
||||
note: "RIFF with WAVE form type.",
|
||||
}),
|
||||
signature({
|
||||
format: "matroska",
|
||||
label: "Matroska/WebM",
|
||||
extensions: ["mkv", "mka", "webm"],
|
||||
mediaTypes: ["video/x-matroska", "video/webm"],
|
||||
offset: 0,
|
||||
bytes: [0x1a, 0x45, 0xdf, 0xa3],
|
||||
confidence: "container",
|
||||
note: "EBML header; DocType distinguishes Matroska/WebM.",
|
||||
}),
|
||||
signature({
|
||||
format: "iso-bmff",
|
||||
label: "ISO Base Media",
|
||||
extensions: ["mp4", "m4a", "mov", "avif", "heic"],
|
||||
mediaTypes: ["video/mp4"],
|
||||
offset: 4,
|
||||
bytes: [0x66, 0x74, 0x79, 0x70],
|
||||
confidence: "container",
|
||||
note: "ftyp box marker; compatible brands distinguish formats.",
|
||||
}),
|
||||
];
|
||||
|
||||
export function identifyBySignature(bytes: Uint8Array): FormatSignature[] {
|
||||
return formatSignatures.filter((item) =>
|
||||
item.bytes.every((expected, index) => {
|
||||
const actual = bytes[item.offset + index];
|
||||
const mask = item.mask?.[index] ?? 0xff;
|
||||
return actual !== undefined && (actual & mask) === (expected & mask);
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -531,6 +531,12 @@ tbody tr:last-child > * {
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.selection-grid > *,
|
||||
.explorer-grid > *,
|
||||
.lab-grid > * {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.selection-grid {
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
margin-bottom: 0.85rem;
|
||||
@@ -642,10 +648,12 @@ tbody tr:last-child > * {
|
||||
|
||||
.route-statuses small,
|
||||
.status {
|
||||
max-width: 100%;
|
||||
border-radius: 999px;
|
||||
padding: 0.22rem 0.42rem;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 750;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.expected-preserved,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"schemaVersion": 1,
|
||||
"id": "de.add-ideas.format-lab",
|
||||
"name": "Format Lab",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"description": "Explore conversion paths and measure information loss locally.",
|
||||
"entry": "./",
|
||||
"icon": "./favicon.svg",
|
||||
@@ -28,6 +28,48 @@
|
||||
"crossOriginIsolated": false,
|
||||
"topLevelContext": false
|
||||
},
|
||||
"io": {
|
||||
"accepts": [
|
||||
{
|
||||
"mediaType": "application/json",
|
||||
"extensions": [".json"]
|
||||
},
|
||||
{
|
||||
"mediaType": "application/x-ndjson",
|
||||
"extensions": [".ndjson"]
|
||||
},
|
||||
{
|
||||
"mediaType": "text/csv",
|
||||
"extensions": [".csv"]
|
||||
},
|
||||
{
|
||||
"mediaType": "application/xml",
|
||||
"extensions": [".xml"]
|
||||
}
|
||||
],
|
||||
"produces": [
|
||||
{
|
||||
"mediaType": "application/json",
|
||||
"extensions": [".json"]
|
||||
},
|
||||
{
|
||||
"mediaType": "application/x-ndjson",
|
||||
"extensions": [".ndjson"]
|
||||
},
|
||||
{
|
||||
"mediaType": "text/csv",
|
||||
"extensions": [".csv"]
|
||||
},
|
||||
{
|
||||
"mediaType": "application/xml",
|
||||
"extensions": [".xml"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"required": [],
|
||||
"optional": []
|
||||
},
|
||||
"privacy": {
|
||||
"processing": "local",
|
||||
"fileUploads": true,
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
export const APP_VERSION = "0.1.0";
|
||||
export const APP_VERSION = "0.2.0";
|
||||
|
||||
@@ -83,6 +83,25 @@ test("shows CSV losses and retains evidence after invalid source", async ({
|
||||
await expect(output).toHaveValue(previous);
|
||||
});
|
||||
|
||||
test("identifies bounded signatures and exposes the portable evidence contract", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page.goto("/deep/nested/format-lab/");
|
||||
await page
|
||||
.locator('input[type="file"]')
|
||||
.first()
|
||||
.setInputFiles("tests/fixtures/signature.pdf");
|
||||
await expect(page.getByText(/PDF \(strong: %PDF- header/u)).toBeVisible();
|
||||
await page.getByText("Portable evidence JSON", { exact: true }).click();
|
||||
const evidence = await page
|
||||
.getByRole("textbox", { name: "Evidence report", exact: true })
|
||||
.inputValue();
|
||||
expect(JSON.parse(evidence)).toMatchObject({
|
||||
contractVersion: 1,
|
||||
provenance: { execution: "local-browser", networkRequired: false },
|
||||
});
|
||||
});
|
||||
|
||||
test("integrates help, themes, identity, headers and offline reload", async ({
|
||||
page,
|
||||
request,
|
||||
@@ -115,7 +134,7 @@ test("integrates help, themes, identity, headers and offline reload", async ({
|
||||
);
|
||||
await expect(manifest.json()).resolves.toMatchObject({
|
||||
id: "de.add-ideas.format-lab",
|
||||
version: "0.1.0",
|
||||
version: "0.2.0",
|
||||
privacy: { processing: "local", telemetry: false },
|
||||
});
|
||||
await context.setOffline(true);
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
test("keeps the primary workspace inside a narrow viewport", async ({
|
||||
page,
|
||||
}) => {
|
||||
await page.goto("/deep/nested/format-lab/");
|
||||
await expect(page.locator("main").first()).toBeVisible();
|
||||
await expect(
|
||||
page.locator("main .loading, main .workbench-loading"),
|
||||
).toHaveCount(0);
|
||||
|
||||
const widths = await page.evaluate(() => ({
|
||||
content: document.documentElement.scrollWidth,
|
||||
viewport: document.documentElement.clientWidth,
|
||||
}));
|
||||
expect(widths.viewport).toBeLessThanOrEqual(430);
|
||||
expect(widths.content).toBeLessThanOrEqual(widths.viewport + 1);
|
||||
});
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
%PDF-1.7
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { findRoutes, properties } from "../../src/lab/catalog";
|
||||
import { parseData, runRoundTrip, serializeData } from "../../src/lab/data";
|
||||
import { parseEvidence, serializeEvidence } from "../../src/lab/evidence";
|
||||
|
||||
const all = properties.data.map((property) => property.id);
|
||||
const sample = JSON.stringify([
|
||||
@@ -29,6 +30,13 @@ describe("structured data lab", () => {
|
||||
property: "metadata",
|
||||
status: "not-tested",
|
||||
});
|
||||
expect(parseEvidence(serializeEvidence(result.evidence))).toEqual(
|
||||
result.evidence,
|
||||
);
|
||||
expect(result.evidence).toMatchObject({
|
||||
contractVersion: 1,
|
||||
provenance: { execution: "local-browser", networkRequired: false },
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes CSV type, null and nested-value changes", () => {
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { exportCorpusFiles } from "../../src/lab/corpus";
|
||||
import {
|
||||
formatSignatures,
|
||||
identifyBySignature,
|
||||
} from "../../src/lab/signatures";
|
||||
|
||||
describe("format signatures and corpus", () => {
|
||||
it("distinguishes masked RIFF forms and container-only evidence", () => {
|
||||
const wav = Uint8Array.of(
|
||||
0x52,
|
||||
0x49,
|
||||
0x46,
|
||||
0x46,
|
||||
0x24,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0x57,
|
||||
0x41,
|
||||
0x56,
|
||||
0x45,
|
||||
);
|
||||
expect(identifyBySignature(wav).map((item) => item.format)).toContain(
|
||||
"wav",
|
||||
);
|
||||
expect(identifyBySignature(wav).map((item) => item.format)).not.toContain(
|
||||
"webp",
|
||||
);
|
||||
expect(
|
||||
identifyBySignature(Uint8Array.of(0x50, 0x4b, 0x03, 0x04))[0],
|
||||
).toMatchObject({ format: "zip", confidence: "container" });
|
||||
});
|
||||
|
||||
it("ships a broad, deterministic structured regression corpus", () => {
|
||||
expect(formatSignatures.length).toBeGreaterThanOrEqual(20);
|
||||
const first = exportCorpusFiles();
|
||||
expect(first).toEqual(exportCorpusFiles());
|
||||
expect(first.map((file) => file.path)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"structured/typed.json",
|
||||
"structured/text-cells.csv",
|
||||
"corpus-manifest.json",
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user