From f2c0fc12d2dca7c9dc2c3393147317168328d050 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Mon, 27 Jul 2026 18:25:47 +0200 Subject: [PATCH] feat: make media inspection progressive --- README.md | 22 +- docs/ARCHITECTURE.md | 35 +- docs/KNOWN_LIMITATIONS.md | 11 +- docs/PERFORMANCE.md | 14 +- package.json | 2 +- playwright.config.ts | 5 + src/App.tsx | 698 ++++++++++++++----- src/app/application-state.ts | 52 +- src/components/CapabilityPanel.tsx | 15 +- src/components/HelpDialog.tsx | 10 +- src/components/Inspector.tsx | 226 ++++-- src/components/MediaBin.tsx | 64 +- src/components/MediaPreview.tsx | 18 +- src/components/QuickConvert.tsx | 164 +++-- src/components/quick-convert-selection.ts | 70 ++ src/ffmpeg/EngineManager.ts | 340 +++++++-- src/index.css | 27 +- src/media/browser-metadata.ts | 282 ++++++++ src/media/index.ts | 1 + tests/browser/application.spec.ts | 33 + tests/browser/browser-helpers.ts | 15 + tests/browser/ffmpeg-runtime.spec.ts | 24 +- tests/browser/fixture-recovery.spec.ts | 17 +- tests/browser/large-media-preview.spec.ts | 20 +- tests/browser/operation-matrix.spec.ts | 13 + tests/browser/performance.spec.ts | 3 +- tests/browser/ui-workflows.spec.ts | 14 +- tests/unit/components/media-bin.test.tsx | 93 +++ tests/unit/components/media-preview.test.tsx | 60 +- tests/unit/components/quick-convert.test.tsx | 60 ++ tests/unit/ffmpeg-engine-arguments.test.ts | 36 + tests/unit/media/browser-metadata.test.ts | 187 +++++ 32 files changed, 2169 insertions(+), 462 deletions(-) create mode 100644 src/components/quick-convert-selection.ts create mode 100644 src/media/browser-metadata.ts create mode 100644 tests/unit/components/media-bin.test.tsx create mode 100644 tests/unit/media/browser-metadata.test.ts diff --git a/README.md b/README.md index 550d4d0..8d718ed 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ under Toolbox Portal with shared shell, theme, help and app switching. Two workflows keep the product intentionally smaller than a nonlinear editor: -- **Quick Convert:** drop/import media, inspect streams, choose a - capability-checked preset/streams, convert, preview when the browser can, and - download. +- **Quick Convert:** drop/import media, play it and read browser-native file + information immediately, then inspect streams on demand when an export needs + authoritative details. - **Edit:** a media bin and sequential non-overlapping clip timeline with non-destructive settings, one export at a time. @@ -225,10 +225,12 @@ profile. `AV_TOOLS_LARGE_MEDIA=/absolute/video.mp4 npm run evidence:large-preview` performs the opt-in large-file regression: the native video element must mount -and advance before bounded inspection finishes, remain the same element after -the authoritative probe completes and produce no browser error. +and advance while detailed inspection remains idle, browser-native +duration/dimensions must appear without loading FFmpeg, and the same element +must survive a subsequently requested authoritative probe. The command runs +that evidence in both Chromium and Firefox. -The latest recorded local Vitest run on 2026-07-26 completed **313 passing +The latest recorded local Vitest run on 2026-07-27 completed **332 passing tests and two conditional skips**. It includes broad structured-command execution against native FFmpeg 7.1.3; the skipped native subtitle burn-in and labeled contact-sheet cases require `subtitles` and `drawtext` filters that are @@ -241,9 +243,11 @@ queue and recovery workflows. The MT core is directly initialized and queried, and completes a real non-empty, re-probeable H.264/AAC conversion; the same path is benchmarked with 2-second 160×90 and 6-second 640×360 inputs. Nested static/Toolbox-context loading is also covered. This evidence is -Chromium-specific: Firefox, Safari, mobile and exhaustive MT operation parity -remain unrecorded. The exact one-run performance record and its limitations are -in [performance observations](docs/PERFORMANCE.md). +Chromium-specific. Firefox now covers the opt-in large-file +play-before-inspection regression and its explicit authoritative-probe +transition; broad Firefox operation parity, Safari, mobile and exhaustive MT +operation parity remain unrecorded. The exact one-run performance record and +its limitations are in [performance observations](docs/PERFORMANCE.md). ## Deployment diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7aa5b00..579729a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -27,8 +27,10 @@ Browser storage boundary ``` - `src/toolbox/` owns the one typed manifest and suite integration. -- `src/media/` normalizes ffprobe JSON, duration/timecode, stream selection, - playback MIME decisions and safe names. +- `src/media/` reads lightweight browser-native metadata, normalizes + authoritative ffprobe JSON, and owns duration/timecode, stream-selection, + playback-MIME and safe-name decisions. Browser metadata and probe data are + deliberately separate models. - `src/project/` owns the versioned, non-destructive project document and migrations. A source is represented by identity/metadata and must be reattached after load; the current application does not persist source bytes. @@ -51,20 +53,35 @@ raw command field. ## FFmpeg lifecycle -1. Inspect secure-context, worker, `crossOriginIsolated` and +1. Import creates an object URL immediately. A detached native media element + reads duration/dimensions where supported; playback and file identity do not + initialize FFmpeg. +2. Detailed inspection remains idle until the user requests it or an operation + requires codecs, complete streams, chapters, tags or dispositions. +3. Inspect secure-context, worker, `crossOriginIsolated` and `SharedArrayBuffer` availability. -2. Select MT only when all MT prerequisites are true (or ST when explicitly +4. Select MT only when all MT prerequisites are true (or ST when explicitly selected). A failed MT load is terminated and retried once with ST. -3. Load wrapper/core assets via base-relative, same-origin URLs. -4. Query version/build configuration/formats/codecs/encoders/filters +5. Load wrapper/core assets via base-relative, same-origin URLs. +6. Query version/build configuration/formats/codecs/encoders/filters sequentially and publish immutable capability state. -5. For one job, validate resource estimates and structured settings, allocate +7. For one job, validate resource estimates and structured settings, allocate a generated directory, mount/write inputs, run a generated argument array, read only declared outputs, and re-probe applicable results. -6. Return result bytes and an export report, then unmount/delete virtual files. -7. Retain the idle engine for the next sequential job; terminate it on mode +8. Return result bytes and an export report, then unmount/delete virtual files. +9. Retain the idle engine for the next sequential job; terminate it on mode changes, unrecoverable errors or cancellation. +A detailed-probe failure changes only the inspection state. The File, object +URL, browser metadata and provisional timeline clip remain available, and the +user can retry. An authoritative probe supersedes browser duration/dimensions +and reconciles an untouched provisional full-source clip. + +The reviewed Emscripten runtime can stall when Firefox probes on a core that +has retained prior application state. Before an explicit Firefox probe, the +manager recreates the identical ST/MT core and reuses only that mode's already +validated capability inventory. Chromium retains the normal warm-core path. + The FFmpeg instance is a single process. `maxConcurrentFFmpegJobs` is one and no probe/export operations overlap on it. The application-facing scheduler is also bounded: it retains at most eight scheduled operations in memory, diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 8ace317..0a192cd 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -45,6 +45,9 @@ FFmpeg output does not remove these product/browser/resource limits. presets from runtime formats/codecs/encoders/filters. - FFmpeg-readable output may be download-only because browser media elements support fewer container/codec combinations. +- Browser-native basic information intentionally omits codecs, complete stream + lists, chapters, tags, subtitles, attachments and dispositions. Features + that need those details request an authoritative local FFprobe pass. - Browser support, codec policy and memory differ across Chromium, Firefox and Safari and across desktop/mobile versions. - The reviewed 0.12.10 core profile passes the non-empty Opus encode/re-probe @@ -83,9 +86,11 @@ FFmpeg output does not remove these product/browser/resource limits. together; development-only smoke artifacts must not be distributed. - Chromium covers a broad real pinned-core ST operation matrix, a real MT H.264/AAC conversion, MT-to-ST fallback, nested static/Toolbox-context - loading and one two-size ST/MT performance baseline. Firefox, Safari, mobile, - exhaustive MT feature parity and repeated performance measurements remain - unrecorded. Live Portal container/header verification is a deployment gate. + loading and one two-size ST/MT performance baseline. Firefox covers the + opt-in large-file browser-metadata/playback-to-authoritative-probe + regression. Broad Firefox operation parity, Safari, mobile, exhaustive MT + feature parity and repeated performance measurements remain unrecorded. Live + Portal container/header verification is a deployment gate. See `docs/FORMAT_SUPPORT.md`, `docs/PERFORMANCE.md`, `docs/PORTAL_REQUIREMENTS.md` and `docs/RELEASE.md` for testable detail. diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index b2bf61b..7b58649 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -80,11 +80,15 @@ frames meet the threshold. The documented ffmpeg.wasm input ceiling is 2 GiB, but real limits can be substantially lower on mobile or memory-constrained browsers. -Source inspection has a bounded, size-aware watchdog: 30 seconds for small -files, a two-MiB-per-second scan budget for larger files, and a five-minute -ceiling. Recognized local audio/video can begin browser playback from its -temporary object URL while that worker inspection continues; completed probe -metadata remains authoritative for editing and export. +Import does not start FFmpeg. Recognized local audio/video can begin browser +playback from its temporary object URL while a native media element reads +lightweight duration/dimensions. Detailed FFprobe inspection starts only when +requested or required for an export and has a bounded, size-aware watchdog: +30 seconds for small files, a two-MiB-per-second budget for larger files, and a +five-minute ceiling. A detailed-inspection timeout is nonfatal; browser +playback and basic file information remain available. Completed probe metadata +remains authoritative for stream mapping, remux safety, chapters, tags, +subtitles/attachments and export validation. Lowering resolution, duration, frame rate, passes and output count reduces work. MT can be faster but consumes more memory/CPU and requires isolation; ST diff --git a/package.json b/package.json index 48882bc..10a7d36 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "test": "vitest run", "test:watch": "vitest", "test:browser": "playwright test", - "evidence:large-preview": "playwright test tests/browser/large-media-preview.spec.ts --project=chromium --workers=1", + "evidence:large-preview": "playwright test tests/browser/large-media-preview.spec.ts --project=chromium --project=firefox-large-media --workers=1", "evidence:opus": "npm run vendor:ffmpeg && npm run vendor:verify-runtime && node scripts/opus-regression.mjs", "test:opus-regression": "npm run vendor:ffmpeg && npm run vendor:verify-runtime && node scripts/opus-regression.mjs --require-pass", "benchmark:browser": "AV_BENCHMARK=1 playwright test tests/browser/performance.spec.ts --project=chromium --workers=1", diff --git a/playwright.config.ts b/playwright.config.ts index 69cdda4..28d7ca6 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -21,5 +21,10 @@ export default defineConfig({ name: 'chromium', use: { ...devices['Desktop Chrome'] }, }, + { + name: 'firefox-large-media', + testMatch: 'large-media-preview.spec.ts', + use: { ...devices['Desktop Firefox'] }, + }, ], }); diff --git a/src/App.tsx b/src/App.tsx index 873c7f9..67fbc78 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,10 +9,7 @@ import { type WorkspaceMode, } from './app/application-state'; import { AppErrorBoundary } from './app/AppErrorBoundary'; -import { - SourceInspectionRegistry, - type SourceInspectionToken, -} from './app/source-inspection-registry'; +import { SourceInspectionRegistry } from './app/source-inspection-registry'; import { FileDropZone } from './components/FileDropZone'; import { MediaBin } from './components/MediaBin'; import { @@ -23,6 +20,7 @@ import { QuickConvert, type QuickExportConfiguration, } from './components/QuickConvert'; +import { defaultQuickStreamSelection } from './components/quick-convert-selection'; import { CapabilityPanel } from './components/CapabilityPanel'; import { JobPanel } from './components/JobPanel'; import { StoragePanel } from './components/StoragePanel'; @@ -58,7 +56,11 @@ import type { EngineState, } from './ffmpeg/ffmpeg.types'; import { EngineError } from './ffmpeg/ffmpeg-error'; -import { normalizeFfprobeReport } from './media'; +import { + BrowserMetadataError, + inspectBrowserMediaMetadata, + normalizeFfprobeReport, +} from './media'; import { createEmptyProject, importProjectDocument, @@ -304,6 +306,74 @@ function resultAsFile(result: ExportResult): File { }); } +function projectWithAssetInspection( + current: AvProjectV1, + asset: ImportedMediaAsset, + probe: MediaProbe, + provisionalDuration: number | undefined +): AvProjectV1 { + if (!current.assets.some((entry) => entry.id === asset.id)) return current; + + let next = projectReducer(current, { + type: 'asset/update', + assetId: asset.id, + changes: { probe, mimeType: asset.file.type }, + updatedAt: timestamp(), + }); + const existingClip = next.timeline.find((clip) => clip.assetId === asset.id); + if ( + existingClip && + provisionalDuration !== undefined && + probe.durationSeconds !== undefined && + existingClip.sourceInSeconds === 0 && + Math.abs(existingClip.sourceOutSeconds - provisionalDuration) < 0.05 && + Math.abs(probe.durationSeconds - provisionalDuration) >= 0.001 + ) { + next = projectReducer(next, { + type: 'clip/update', + clipId: existingClip.id, + changes: { sourceOutSeconds: probe.durationSeconds }, + updatedAt: timestamp(), + }); + } else if ( + !existingClip && + probe.durationSeconds !== undefined && + Number.isFinite(probe.durationSeconds) && + probe.durationSeconds >= 0.001 + ) { + next = projectReducer(next, { + type: 'clip/add', + clip: { + id: crypto.randomUUID(), + assetId: asset.id, + sourceInSeconds: 0, + sourceOutSeconds: probe.durationSeconds, + audio: { enabled: true }, + video: { enabled: true }, + }, + updatedAt: timestamp(), + }); + } + + if (next.chapters.length === 0 && probe.chapters.length > 0) { + next = projectReducer(next, { + type: 'chapters/set', + chapters: probe.chapters.map((chapter) => ({ + id: `${asset.id}-chapter-${chapter.id}`, + startSeconds: chapter.startSeconds, + endSeconds: chapter.endSeconds, + title: chapter.title ?? `Chapter ${chapter.id + 1}`, + ...(chapter.timeBase ? { timeBase: chapter.timeBase } : {}), + ...(Object.keys(chapter.tags).length > 0 + ? { metadata: chapter.tags } + : {}), + })), + updatedAt: timestamp(), + }); + } + return next; +} + function concatSourceForClip( clip: TimelineClip, asset: ImportedMediaAsset, @@ -567,6 +637,14 @@ export function AvToolsApplication() { const sourceProbeTail = useRef>(Promise.resolve()); const sourceProbePendingRef = useRef(0); const sourceInspectionRegistry = useRef(new SourceInspectionRegistry()); + const sourceInspectionPromises = useRef( + new Map< + string, + { readonly objectUrl: string; readonly promise: Promise } + >() + ); + const browserMetadataControllers = useRef(new Map()); + const browserMetadataClipTail = useRef>(Promise.resolve()); const planExecutionTail = useRef>(Promise.resolve()); const scheduledPlanCountRef = useRef(0); const scheduledWorkRef = useRef(new Map()); @@ -717,9 +795,14 @@ export function AvToolsApplication() { useEffect(() => { const inspectionRegistry = sourceInspectionRegistry.current; + const metadataControllers = browserMetadataControllers.current; return () => { const cancelActiveInspection = inspectionRegistry.clear(); if (cancelActiveInspection) void engineManager.cancelActive(); + for (const controller of metadataControllers.values()) { + controller.abort(); + } + metadataControllers.clear(); for (const asset of assetsRef.current) { URL.revokeObjectURL(asset.objectUrl); } @@ -734,11 +817,13 @@ export function AvToolsApplication() { const updateAssetState = useCallback( (id: string, changes: Partial) => { - setAssets((current) => - current.map((asset) => + setAssets((current) => { + const next = current.map((asset) => asset.id === id ? { ...asset, ...changes } : asset - ) - ); + ); + assetsRef.current = next; + return next; + }); }, [] ); @@ -766,117 +851,189 @@ export function AvToolsApplication() { [] ); + const addDefaultClip = useCallback( + ( + current: AvProjectV1, + assetId: string, + durationSeconds: number | undefined + ): AvProjectV1 => { + if ( + durationSeconds === undefined || + !Number.isFinite(durationSeconds) || + durationSeconds < 0.001 || + current.timeline.some((clip) => clip.assetId === assetId) + ) { + return current; + } + return projectReducer(current, { + type: 'clip/add', + clip: { + id: crypto.randomUUID(), + assetId, + sourceInSeconds: 0, + sourceOutSeconds: durationSeconds, + audio: { enabled: true }, + video: { enabled: true }, + }, + updatedAt: timestamp(), + }); + }, + [] + ); + + const readBrowserMetadata = useCallback( + (asset: ImportedMediaAsset) => { + browserMetadataControllers.current.get(asset.id)?.abort(); + const controller = new AbortController(); + browserMetadataControllers.current.set(asset.id, controller); + return inspectBrowserMediaMetadata(asset.file, asset.objectUrl, { + signal: controller.signal, + }) + .then((browserMetadata) => { + const currentAsset = assetsRef.current.find( + (entry) => + entry.id === asset.id && entry.objectUrl === asset.objectUrl + ); + if (!currentAsset || controller.signal.aborted) return; + updateAssetState(asset.id, { + browserMetadata, + browserMetadataError: undefined, + }); + return browserMetadata; + }) + .catch((error: unknown) => { + if ( + controller.signal.aborted || + (error instanceof BrowserMetadataError && error.code === 'aborted') + ) { + return; + } + const currentAsset = assetsRef.current.find( + (entry) => + entry.id === asset.id && entry.objectUrl === asset.objectUrl + ); + if (!currentAsset) return; + updateAssetState(asset.id, { + browserMetadataError: + error instanceof Error + ? error.message + : 'The browser could not read basic media metadata.', + }); + return undefined; + }) + .finally(() => { + if (browserMetadataControllers.current.get(asset.id) === controller) { + browserMetadataControllers.current.delete(asset.id); + } + }); + }, + [updateAssetState] + ); + const inspectAsset = useCallback( - async ( - asset: ImportedMediaAsset, - existingAsset: boolean, - token: SourceInspectionToken - ) => { - if (!sourceInspectionRegistry.current.isCurrent(token)) return; - updateAssetState(asset.id, { + (requestedAsset: ImportedMediaAsset): Promise => { + const currentAsset = assetsRef.current.find( + (entry) => + entry.id === requestedAsset.id && + entry.objectUrl === requestedAsset.objectUrl + ); + if (!currentAsset) { + return Promise.reject( + new EngineError( + 'cancelled', + 'The media source is no longer attached.' + ) + ); + } + if (currentAsset.probe) return Promise.resolve(currentAsset.probe); + + const active = sourceInspectionPromises.current.get(currentAsset.id); + if (active?.objectUrl === currentAsset.objectUrl) return active.promise; + + const token = sourceInspectionRegistry.current.register(currentAsset.id); + updateAssetState(currentAsset.id, { phase: 'probing', error: undefined, }); - try { - const probeResult = await runSourceProbe(async () => { - if (!sourceInspectionRegistry.current.activate(token)) { - return undefined; - } - try { - return await engineManager.probe( - asset.file, - undefined, - token.signal - ); - } finally { - sourceInspectionRegistry.current.finish(token); - } - }); - if ( - !probeResult || - !sourceInspectionRegistry.current.isCurrent(token) - ) { - return; - } - const probe = normalizeFfprobeReport(probeResult.json); - const rawProbe = `${JSON.stringify(probeResult.json, null, 2)}\n`; - const importedChapters = probe.chapters.map((chapter) => ({ - id: `${asset.id}-chapter-${chapter.id}`, - startSeconds: chapter.startSeconds, - endSeconds: chapter.endSeconds, - title: chapter.title ?? `Chapter ${chapter.id + 1}`, - ...(chapter.timeBase ? { timeBase: chapter.timeBase } : {}), - ...(Object.keys(chapter.tags).length > 0 - ? { metadata: chapter.tags } - : {}), - })); - if (!sourceInspectionRegistry.current.isCurrent(token)) return; - updateAssetState(asset.id, { phase: 'ready', probe, rawProbe }); - setProject((current) => { - if ( - !sourceInspectionRegistry.current.isCurrent(token) || - !current.assets.some((entry) => entry.id === asset.id) - ) { - return current; - } - let next = projectReducer(current, { - type: 'asset/update', - assetId: asset.id, - changes: { probe, mimeType: asset.file.type }, - updatedAt: timestamp(), + + const promise = (async (): Promise => { + try { + const probeResult = await runSourceProbe(async () => { + if (!sourceInspectionRegistry.current.activate(token)) { + throw new EngineError( + 'cancelled', + 'The detailed media inspection was superseded.' + ); + } + try { + return await engineManager.probe( + currentAsset.file, + undefined, + token.signal + ); + } finally { + sourceInspectionRegistry.current.finish(token); + } }); - if ( - existingAsset && - next.assets.find((entry) => entry.id === asset.id)?.sourceStatus === - 'missing' - ) { - next = projectReducer(next, { - type: 'asset/set-source-status', - assetId: asset.id, - sourceStatus: 'attached', - updatedAt: timestamp(), + if (!sourceInspectionRegistry.current.isCurrent(token)) { + throw new EngineError( + 'cancelled', + 'The detailed media inspection was superseded.' + ); + } + + const probe = normalizeFfprobeReport(probeResult.json); + const rawProbe = `${JSON.stringify(probeResult.json, null, 2)}\n`; + const provisionalDuration = assetsRef.current.find( + (entry) => + entry.id === currentAsset.id && + entry.objectUrl === currentAsset.objectUrl + )?.browserMetadata?.durationSeconds; + updateAssetState(currentAsset.id, { + phase: 'ready', + probe, + rawProbe, + error: undefined, + }); + setProject((current) => { + if (!sourceInspectionRegistry.current.isCurrent(token)) { + return current; + } + const next = projectWithAssetInspection( + current, + currentAsset, + probe, + provisionalDuration + ); + projectRef.current = next; + return next; + }); + return probe; + } catch (error) { + if (sourceInspectionRegistry.current.isCurrent(token)) { + updateAssetState(currentAsset.id, { + phase: 'error', + error: + error instanceof Error + ? error.message + : 'Detailed media inspection failed.', }); } - const hasClip = next.timeline.some( - (clip) => clip.assetId === asset.id + throw error; + } finally { + const activeInspection = sourceInspectionPromises.current.get( + currentAsset.id ); - if ( - !hasClip && - probe.durationSeconds !== undefined && - probe.durationSeconds >= 0.001 - ) { - next = projectReducer(next, { - type: 'clip/add', - clip: { - id: crypto.randomUUID(), - assetId: asset.id, - sourceInSeconds: 0, - sourceOutSeconds: probe.durationSeconds, - audio: { enabled: true }, - video: { enabled: true }, - }, - updatedAt: timestamp(), - }); + if (activeInspection?.objectUrl === currentAsset.objectUrl) { + sourceInspectionPromises.current.delete(currentAsset.id); } - if (next.chapters.length === 0 && importedChapters.length > 0) { - next = projectReducer(next, { - type: 'chapters/set', - chapters: importedChapters, - updatedAt: timestamp(), - }); - } - return next; - }); - } catch (error) { - if (!sourceInspectionRegistry.current.isCurrent(token)) return; - updateAssetState(asset.id, { - phase: 'error', - error: - error instanceof Error - ? error.message - : 'The file could not be inspected.', - }); - } + } + })(); + sourceInspectionPromises.current.set(currentAsset.id, { + objectUrl: currentAsset.objectUrl, + promise, + }); + return promise; }, [runSourceProbe, updateAssetState] ); @@ -919,16 +1076,12 @@ export function AvToolsApplication() { .map((issue) => issue.message) .join( '\n' - )}\n\n${importDecision.estimateDisclaimer}\n\nImport and inspect anyway?` + )}\n\n${importDecision.estimateDisclaimer}\n\nImport anyway?` ) ) { return; } - const pending: Array<{ - asset: ImportedMediaAsset; - existingAsset: boolean; - token: SourceInspectionToken; - }> = []; + const pending: ImportedMediaAsset[] = []; const claimedIds = new Set(assetsRef.current.map((asset) => asset.id)); for (const file of files) { @@ -963,11 +1116,15 @@ export function AvToolsApplication() { if (reattach && match) { asset.id = match.entry.id; claimedIds.add(match.entry.id); - pending.push({ - asset, - existingAsset: true, - token: sourceInspectionRegistry.current.register(asset.id), - }); + setProject((current) => + projectReducer(current, { + type: 'asset/set-source-status', + assetId: asset.id, + sourceStatus: 'attached', + updatedAt: timestamp(), + }) + ); + pending.push(asset); } else { setProject((current) => projectReducer(current, { @@ -983,31 +1140,53 @@ export function AvToolsApplication() { updatedAt: timestamp(), }) ); - pending.push({ - asset, - existingAsset: false, - token: sourceInspectionRegistry.current.register(asset.id), - }); + pending.push(asset); } } setAssets((current) => { - const next = [...current, ...pending.map(({ asset }) => asset)]; + const next = [...current, ...pending]; assetsRef.current = next; return next; }); - setSelectedAssetId((current) => current ?? pending[0]?.asset.id); - for (const item of pending) { - await inspectAsset(item.asset, item.existingAsset, item.token); + setSelectedAssetId((current) => current ?? pending[0]?.id); + for (const asset of pending) { + const metadata = readBrowserMetadata(asset); + browserMetadataClipTail.current = browserMetadataClipTail.current + .then(async () => { + const resolved = await metadata; + if (!resolved) return; + setProject((current) => { + if (!current.assets.some((entry) => entry.id === asset.id)) { + return current; + } + const next = addDefaultClip( + current, + asset.id, + resolved.durationSeconds + ); + projectRef.current = next; + return next; + }); + }) + .catch(() => undefined); } }, - [inspectAsset, project.assets, resourceContext.policy] + [ + addDefaultClip, + project.assets, + readBrowserMetadata, + resourceContext.policy, + ] ); const removeAsset = useCallback( (id: string) => { const asset = assets.find((entry) => entry.id === id); if (asset) URL.revokeObjectURL(asset.objectUrl); + browserMetadataControllers.current.get(id)?.abort(); + browserMetadataControllers.current.delete(id); + sourceInspectionPromises.current.delete(id); const cancelActiveInspection = sourceInspectionRegistry.current.remove(id); setAssets((current) => { @@ -1048,6 +1227,33 @@ export function AvToolsApplication() { [applyProjectAction, assets, selectedAssetId] ); + const requestDetailedInspection = useCallback( + (assetId: string) => { + const asset = assetsRef.current.find((entry) => entry.id === assetId); + if (!asset) return; + setOperationError(undefined); + if ( + scheduledPlanCountRef.current > 0 || + (engineManager.state.status === 'running' && + sourceProbePendingRef.current === 0) + ) { + setOperationError( + 'Wait for the active export or analysis job before inspecting another source.' + ); + return; + } + void inspectAsset(asset).catch((error: unknown) => { + if (error instanceof EngineError && error.code === 'cancelled') return; + setOperationError( + error instanceof Error + ? `Detailed inspection failed: ${error.message}` + : 'Detailed inspection failed.' + ); + }); + }, + [inspectAsset] + ); + const selectedAsset = assets.find((asset) => asset.id === selectedAssetId); const selectedClip = project.timeline.find( (clip) => clip.id === selectedClipId @@ -1625,9 +1831,18 @@ export function AvToolsApplication() { const exportQuick = useCallback( async (configuration: QuickExportConfiguration) => { - if (!selectedAsset?.probe) return; + if (!selectedAsset) return; const jobId = crypto.randomUUID(); try { + const probe = + selectedAsset.probe ?? (await inspectAsset(selectedAsset)); + const streamSelection = configuration.selectDetectedStreams + ? defaultQuickStreamSelection( + probe, + configuration.operation, + configuration.preset + ) + : configuration.streamSelection; let plan: FFmpegCommandPlan; if (configuration.operation === 'remux') { const container = configuration.remuxContainer ?? 'matroska'; @@ -1646,7 +1861,7 @@ export function AvToolsApplication() { ? 'ogg' : container, description: { - streams: selectedAsset.probe.streams.flatMap((stream) => + streams: probe.streams.flatMap((stream) => stream.type === 'video' || stream.type === 'audio' || stream.type === 'subtitle' || @@ -1661,13 +1876,13 @@ export function AvToolsApplication() { ] : [] ), - hasChapters: selectedAsset.probe.chapters.length > 0, - hasMetadata: Object.keys(selectedAsset.probe.tags).length > 0, + hasChapters: probe.chapters.length > 0, + hasMetadata: Object.keys(probe.tags).length > 0, }, - streamSelection: configuration.streamSelection, + streamSelection, metadataPolicy: configuration.removeMetadata ? 'remove' : 'copy', chapterPolicy: configuration.removeChapters ? 'remove' : 'keep', - expectedDurationSeconds: selectedAsset.probe.durationSeconds, + expectedDurationSeconds: probe.durationSeconds, }); } else { if (!configuration.preset) return; @@ -1683,8 +1898,8 @@ export function AvToolsApplication() { metadataPolicy: configuration.removeMetadata ? 'remove' : 'copy', chapterPolicy: configuration.removeChapters ? 'remove' : 'keep', }, - streamSelection: configuration.streamSelection, - expectedDurationSeconds: selectedAsset.probe.durationSeconds, + streamSelection, + expectedDurationSeconds: probe.durationSeconds, }); } await runPlan(plan, [selectedAsset.file], plan.operation); @@ -1698,7 +1913,7 @@ export function AvToolsApplication() { ); } }, - [runPlan, selectedAsset] + [inspectAsset, runPlan, selectedAsset] ); const extractThumbnail = useCallback( @@ -1763,14 +1978,19 @@ export function AvToolsApplication() { const createPreviewProxy = useCallback( async (asset: ImportedMediaAsset | undefined) => { - const duration = asset?.probe?.durationSeconds; - if (!asset || duration === undefined || duration <= 0) return; + if (!asset) return; try { + const probe = asset.probe ?? (await inspectAsset(asset)); + const duration = probe.durationSeconds; + if (duration === undefined || duration <= 0) { + throw new Error( + 'The source duration is unavailable after detailed inspection.' + ); + } const includeAudio = - asset.probe?.streams.some((stream) => stream.type === 'audio') === - true; + probe.streams.some((stream) => stream.type === 'audio') === true; const hasVideo = - asset.probe?.streams.some( + probe.streams.some( (stream) => stream.type === 'video' && stream.disposition.attached_pic !== true @@ -1861,6 +2081,7 @@ export function AvToolsApplication() { } }, [ + inspectAsset, previewProxySettings, project.id, refreshResults, @@ -1888,7 +2109,39 @@ export function AvToolsApplication() { } const timelineContainer = preset.container as 'mp4' | 'webm' | 'matroska'; try { - const enabledSubtitleTracks = project.subtitles.filter( + const timelineAssetIds = [ + ...new Set(project.timeline.map((clip) => clip.assetId)), + ]; + const timelineInspections = new Map< + string, + { readonly asset: ImportedMediaAsset; readonly probe: MediaProbe } + >(); + for (const assetId of timelineAssetIds) { + const asset = assetsRef.current.find((entry) => entry.id === assetId); + if (!asset) { + throw new Error( + 'Reattach every timeline source before rendering the project.' + ); + } + const probe = asset.probe ?? (await inspectAsset(asset)); + timelineInspections.set(assetId, { asset, probe }); + } + let inspectedProject = projectRef.current; + for (const { asset, probe } of timelineInspections.values()) { + const latestAsset = + assetsRef.current.find( + (entry) => + entry.id === asset.id && entry.objectUrl === asset.objectUrl + ) ?? asset; + inspectedProject = projectWithAssetInspection( + inspectedProject, + latestAsset, + probe, + latestAsset.browserMetadata?.durationSeconds + ); + } + projectRef.current = inspectedProject; + const enabledSubtitleTracks = inspectedProject.subtitles.filter( (entry) => entry.mode !== 'exclude' ); const timelineStepCount = @@ -1897,13 +2150,15 @@ export function AvToolsApplication() { enabledSubtitleTracks.filter( (entry) => (entry.offsetSeconds ?? 0) !== 0 ).length + - (project.chapters.length > 0 ? 1 : 0); + (inspectedProject.chapters.length > 0 ? 1 : 0); await runWorkflow( 'Render timeline project', timelineStepCount, async (executeStep) => { - const resolved = project.timeline.map((clip, index) => { - const asset = assets.find((entry) => entry.id === clip.assetId); + const resolved = inspectedProject.timeline.map((clip, index) => { + const asset = assetsRef.current.find( + (entry) => entry.id === clip.assetId + ); const video = asset?.probe?.streams.find( (stream) => stream.type === 'video' && @@ -2001,7 +2256,7 @@ export function AvToolsApplication() { sourceDurationSeconds: asset.probe?.durationSeconds, hasVideo: true, hasAudio: - project.output.audioEnabled && + inspectedProject.output.audioEnabled && clip.audio?.enabled !== false && asset.probe?.streams.some( (stream) => stream.type === 'audio' @@ -2035,9 +2290,13 @@ export function AvToolsApplication() { const even = (value: number) => Math.max(2, Math.round(value / 2) * 2); const requestedWidth = - project.output.width ?? timelineVideoPreset.width ?? undefined; + inspectedProject.output.width ?? + timelineVideoPreset.width ?? + undefined; const requestedHeight = - project.output.height ?? timelineVideoPreset.height ?? undefined; + inspectedProject.output.height ?? + timelineVideoPreset.height ?? + undefined; let outputWidth: number; let outputHeight: number; if (requestedWidth && requestedHeight) { @@ -2064,7 +2323,7 @@ export function AvToolsApplication() { ), ]; if ( - project.output.frameRate === undefined && + inspectedProject.output.frameRate === undefined && timelineVideoPreset.frameRate === undefined && requestedClipFrameRates.length > 1 ) { @@ -2073,28 +2332,34 @@ export function AvToolsApplication() { ); } const frameRate = - project.output.frameRate ?? + inspectedProject.output.frameRate ?? timelineVideoPreset.frameRate ?? requestedClipFrameRates[0] ?? first.sourceFrameRate ?? 30; const sampleRate = - project.output.sampleRate ?? preset.audio?.sampleRate ?? 48_000; + inspectedProject.output.sampleRate ?? + preset.audio?.sampleRate ?? + 48_000; const channelLayout = - project.output.channelLayout ?? + inspectedProject.output.channelLayout ?? (preset.audio?.channels === 1 ? 'mono' : preset.audio?.channels === 6 ? '5.1' : 'stereo'); const missingAudioPolicy = - !project.output.audioEnabled || !preset.audio + !inspectedProject.output.audioEnabled || !preset.audio ? ('drop-all' as const) - : (project.output.missingAudioPolicy ?? 'insert-silence'); - const outputFileName = safeExportFileName(project.output.fileName, { - extension: preset.fileExtension, - fallback: 'timeline', - }); + : (inspectedProject.output.missingAudioPolicy ?? + 'insert-silence'); + const outputFileName = safeExportFileName( + inspectedProject.output.fileName, + { + extension: preset.fileExtension, + fallback: 'timeline', + } + ); const plan = buildTimelineExportPlan({ jobId: crypto.randomUUID(), clips: resolved.map(({ clip }) => clip), @@ -2124,7 +2389,7 @@ export function AvToolsApplication() { 0 ); - const activeSubtitleTracks = project.subtitles + const activeSubtitleTracks = inspectedProject.subtitles .filter((entry) => entry.mode !== 'exclude') .sort((left, right) => { if (left.mode === right.mode) return 0; @@ -2165,7 +2430,7 @@ export function AvToolsApplication() { } const mediaSource = { - id: `timeline-${project.id}`, + id: `timeline-${inspectedProject.id}`, sourceIndex: 0, fileName: currentFile.name, }; @@ -2225,16 +2490,16 @@ export function AvToolsApplication() { currentFile = resultAsFile(subtitledResult); } - if (project.chapters.length > 0) { + if (inspectedProject.chapters.length > 0) { const chaptered = await executeStep( buildChapterMuxPlan({ jobId: crypto.randomUUID(), source: { - id: `timeline-${project.id}`, + id: `timeline-${inspectedProject.id}`, sourceIndex: 0, fileName: currentFile.name, }, - chapters: project.chapters.map((chapter) => ({ + chapters: inspectedProject.chapters.map((chapter) => ({ id: chapter.id, title: chapter.title, startSeconds: chapter.startSeconds, @@ -2259,12 +2524,14 @@ export function AvToolsApplication() { currentFile = resultAsFile(chapteredResult); } - const metadataEdits = metadataEditsFromProject(project.metadata); + const metadataEdits = metadataEditsFromProject( + inspectedProject.metadata + ); const finalized = await executeStep( buildMetadataEditPlan({ jobId: crypto.randomUUID(), source: { - id: `timeline-${project.id}`, + id: `timeline-${inspectedProject.id}`, sourceIndex: 0, fileName: currentFile.name, }, @@ -2295,7 +2562,7 @@ export function AvToolsApplication() { ); } }, [ - assets, + inspectAsset, loudnessByAsset, project, refreshResults, @@ -2307,6 +2574,11 @@ export function AvToolsApplication() { const resetProject = useCallback(() => { for (const asset of assets) URL.revokeObjectURL(asset.objectUrl); + for (const controller of browserMetadataControllers.current.values()) { + controller.abort(); + } + browserMetadataControllers.current.clear(); + sourceInspectionPromises.current.clear(); const cancelActiveInspection = sourceInspectionRegistry.current.clear(); if (cancelActiveInspection) void engineManager.cancelActive(); assetsRef.current = []; @@ -2345,6 +2617,11 @@ export function AvToolsApplication() { try { const imported = importProjectDocument(await file.text()); for (const asset of assets) URL.revokeObjectURL(asset.objectUrl); + for (const controller of browserMetadataControllers.current.values()) { + controller.abort(); + } + browserMetadataControllers.current.clear(); + sourceInspectionPromises.current.clear(); const cancelActiveInspection = sourceInspectionRegistry.current.clear(); if (cancelActiveInspection) void engineManager.cancelActive(); assetsRef.current = []; @@ -3578,10 +3855,7 @@ export function AvToolsApplication() { : requirementsForUserPreset(preset); const availability = activeCapabilities ? presetAvailability({ requirements }, activeCapabilities) - : { - status: 'unavailable' as const, - reasons: ['Initialize the engine to verify this preset.'], - }; + : undefined; const opusBlocked = preset.audio?.codec === 'libopus' && !FFMPEG_OPUS_REGRESSION_VERIFIED; return { @@ -3590,11 +3864,12 @@ export function AvToolsApplication() { fileExtension: preset.fileExtension, supportsAudio: preset.audio !== undefined, supportsVideo: preset.video !== undefined, - ...(availability.status === 'unavailable' || opusBlocked + ...(availability?.status === 'unavailable' || opusBlocked ? { disabledReason: opusBlocked ? 'Opus output is disabled until the reviewed core passes the non-empty ST/MT regression encode and re-probe gate.' - : availability.reasons.join(' '), + : (availability?.reasons.join(' ') ?? + 'The selected preset is unavailable.'), } : {}), }; @@ -3617,6 +3892,10 @@ export function AvToolsApplication() { : !selectedTimelinePreset?.video ? 'Choose a video export preset.' : selectedTimelinePresetOption?.disabledReason; + const timelineNeedsInspection = project.timeline.some((clip) => { + const asset = assets.find((entry) => entry.id === clip.assetId); + return asset !== undefined && asset.probe === undefined; + }); const anyAccuratePreset = advancedPresetOptions.some( (preset) => !preset.disabledReason @@ -3931,6 +4210,8 @@ export function AvToolsApplication() { scheduledPlanCount >= 8; const sourceManagementBusy = sourceProbeBusy || scheduledPlanCount > 0 || waveformProgress !== undefined; + const sourceRemovalBusy = + scheduledPlanCount > 0 || waveformProgress !== undefined; const editWorkspace = ( <> @@ -3993,6 +4274,9 @@ export function AvToolsApplication() { assets={assets} selectedId={selectedAssetId} onSelect={setSelectedAssetId} + onInspect={requestDetailedInspection} + inspectDisabled={sourceManagementBusy} + removeDisabled={sourceRemovalBusy} onRemove={removeAsset} /> @@ -4059,6 +4343,7 @@ export function AvToolsApplication() { } onSubtitlesChange={changeSubtitles} onSubtitleImport={importSubtitleFile} + onInspectDetails={requestDetailedInspection} onThumbnail={extractThumbnail} currentPreviewTimeSeconds={ inspectedClipAsset @@ -4114,6 +4399,42 @@ export function AvToolsApplication() { if (selectedClipId === clipId) setSelectedClipId(undefined); }} /> + {inspectedClipAsset && !inspectedClipAsset.probe ? ( +
+ +
+ + {inspectedClipAsset.phase === 'probing' + ? 'Inspecting source details…' + : 'Advanced tools need source details.'} + + + Playback and basic metadata stay available while codecs, streams + and chapters are inspected locally. + +
+ {inspectedClipAsset.phase !== 'probing' ? ( + + ) : null} +
+ ) : null} {engineState.status === 'running' || scheduledPlanCount > 0 ? 'Add timeline to queue' - : 'Render timeline'} + : timelineNeedsInspection + ? 'Inspect & render timeline' + : 'Render timeline'} @@ -4632,6 +4955,9 @@ export function AvToolsApplication() { assets={assets} selectedId={selectedAssetId} onSelect={setSelectedAssetId} + onInspect={requestDetailedInspection} + inspectDisabled={sourceManagementBusy} + removeDisabled={sourceRemovalBusy} onRemove={removeAsset} /> @@ -4645,6 +4971,9 @@ export function AvToolsApplication() { engineState.status === 'running' || scheduledPlanCount > 0 } presets={[...BUILT_IN_PRESETS, ...userPresets]} + onInspect={() => { + if (selectedAsset) requestDetailedInspection(selectedAsset.id); + }} onExport={exportQuick} />
@@ -4722,9 +5051,18 @@ export function AvToolsApplication() { preference={enginePreference} resourcePolicy={resourceContext.policy} deviceMemoryGiB={resourceContext.deviceMemoryGiB} + disabled={sourceManagementBusy} onPreferenceChange={(preference) => { - setEnginePreference(preference); - engineManager.setPreference(preference); + try { + engineManager.setPreference(preference); + setEnginePreference(preference); + } catch (error) { + setOperationError( + error instanceof Error + ? error.message + : 'The engine mode could not be changed.' + ); + } }} onInitialize={() => { if (engineManager.state.status === 'error') { diff --git a/src/app/application-state.ts b/src/app/application-state.ts index 5ac1851..c9c0d8f 100644 --- a/src/app/application-state.ts +++ b/src/app/application-state.ts @@ -1,12 +1,21 @@ -import type { MediaProbe } from '../media'; +import type { BrowserMediaMetadata, MediaProbe } from '../media'; -export type AssetPhase = 'queued' | 'probing' | 'ready' | 'error'; +/** + * State of the optional authoritative FFprobe inspection. + * + * The File and its object URL are usable in every phase. In particular, + * `error` means only that detailed metadata is unavailable; it never marks the + * local source or browser playback as failed. + */ +export type AssetPhase = 'idle' | 'probing' | 'ready' | 'error'; export interface ImportedMediaAsset { id: string; file: File; objectUrl: string; phase: AssetPhase; + browserMetadata?: BrowserMediaMetadata; + browserMetadataError?: string; probe?: MediaProbe; rawProbe?: string; error?: string; @@ -29,10 +38,47 @@ export function createAsset(file: File): ImportedMediaAsset { id: crypto.randomUUID(), file, objectUrl: URL.createObjectURL(file), - phase: 'queued', + phase: 'idle', }; } +export function assetDurationSeconds( + asset: ImportedMediaAsset | undefined +): number | undefined { + return ( + asset?.probe?.durationSeconds ?? asset?.browserMetadata?.durationSeconds + ); +} + +export function assetHasVideo( + asset: ImportedMediaAsset | undefined +): boolean | undefined { + if (asset?.probe) { + return asset.probe.streams.some( + (stream) => + stream.type === 'video' && stream.disposition.attached_pic !== true + ); + } + if (asset?.browserMetadata?.hasVideo !== undefined) { + return asset.browserMetadata.hasVideo; + } + if (asset?.browserMetadata?.kind === 'video') return true; + if (asset?.browserMetadata?.kind === 'audio') return false; + return undefined; +} + +export function assetHasAudio( + asset: ImportedMediaAsset | undefined +): boolean | undefined { + if (asset?.probe) { + return asset.probe.streams.some((stream) => stream.type === 'audio'); + } + if (asset?.browserMetadata?.hasAudio !== undefined) { + return asset.browserMetadata.hasAudio; + } + return asset?.browserMetadata?.kind === 'audio' ? true : undefined; +} + export function formatBytes(bytes: number): string { if (!Number.isFinite(bytes) || bytes < 0) return 'Unknown size'; if (bytes < 1_000) return `${bytes} B`; diff --git a/src/components/CapabilityPanel.tsx b/src/components/CapabilityPanel.tsx index fdfde15..aea4036 100644 --- a/src/components/CapabilityPanel.tsx +++ b/src/components/CapabilityPanel.tsx @@ -12,6 +12,7 @@ export interface CapabilityPanelProps { preference: EnginePreference; resourcePolicy: ResourcePolicy; deviceMemoryGiB?: number; + disabled?: boolean; onPreferenceChange: (preference: EnginePreference) => void; onInitialize: () => void; } @@ -21,6 +22,7 @@ export function CapabilityPanel({ preference, resourcePolicy, deviceMemoryGiB, + disabled = false, onPreferenceChange, onInitialize, }: CapabilityPanelProps) { @@ -58,7 +60,12 @@ export function CapabilityPanel({ Engine mode