Files
av-tools/src/commands/thumbnails.ts

678 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import {
createPlannedInput,
createPlannedOutput,
diagnostic,
normalizeSeconds,
safePathComponent,
type SourceReference,
} from './command-utils';
import {
GENERATED_OUTPUT_LIMIT_EXPLANATION,
MAX_GENERATED_OUTPUT_FILES,
} from '../limits';
import { freezeCommandPlan, type FFmpegCommandPlan } from './command-plan';
export type ImageFormat = 'png' | 'jpeg' | 'webp';
export const MAX_SCENE_CHANGE_FRAMES = 100;
export const MIN_SCENE_CHANGE_THRESHOLD = 0.01;
export const MAX_SCENE_CHANGE_THRESHOLD = 0.95;
export interface SceneChangeFrame {
/** Exact source presentation timestamp in the source video time base. */
readonly pts: number;
readonly timeSeconds: number;
}
export interface ThumbnailSize {
readonly width?: number;
readonly height?: number;
readonly maintainAspect?: boolean;
}
export interface SingleThumbnailOptions {
readonly jobId: string;
readonly source: SourceReference;
readonly timeSeconds: number;
readonly size?: ThumbnailSize;
readonly format: ImageFormat;
readonly quality?: number;
}
export interface ThumbnailSeriesOptions {
readonly jobId: string;
readonly source: SourceReference;
readonly durationSeconds: number;
readonly timesSeconds?: readonly number[];
readonly intervalSeconds?: number;
readonly count?: number;
readonly size?: ThumbnailSize;
readonly format: ImageFormat;
readonly quality?: number;
}
export interface SceneChangeAnalysisOptions {
readonly jobId: string;
readonly source: SourceReference;
readonly durationSeconds: number;
readonly threshold: number;
readonly maxFrames: number;
}
export interface SceneThumbnailSeriesOptions extends Omit<
ThumbnailSeriesOptions,
'timesSeconds' | 'intervalSeconds' | 'count'
> {
readonly frames: readonly SceneChangeFrame[];
}
export interface ContactSheetOptions {
readonly jobId: string;
readonly source: SourceReference;
readonly durationSeconds: number;
readonly frameCount: number;
readonly rows: number;
readonly columns: number;
readonly cellWidth: number;
readonly spacing?: number;
readonly background?: string;
readonly maintainAspect?: boolean;
readonly timestampLabels?: boolean;
readonly sourceLabel?: boolean;
readonly format: ImageFormat;
readonly quality?: number;
readonly drawtextAvailable?: boolean;
/** Reviewed local font mounted as a second, read-only job input. */
readonly font?: SourceReference;
/** Exact frames returned by a preceding bounded scene-change analysis. */
readonly sceneFrames?: readonly SceneChangeFrame[];
}
export function buildSingleThumbnailPlan(
options: SingleThumbnailOptions
): FFmpegCommandPlan {
if (!Number.isFinite(options.timeSeconds) || options.timeSeconds < 0) {
throw new RangeError('Thumbnail time must be nonnegative');
}
const input = createPlannedInput(options.jobId, options.source);
const extension = extensionForImageFormat(options.format);
const output = createPlannedOutput(options.jobId, {
baseName: options.source.fileName,
operation: 'thumbnail',
suffix: `${Math.round(options.timeSeconds * 1000)}ms`,
extension,
role: 'thumbnail',
});
const args = [
'-ss',
normalizeSeconds(options.timeSeconds),
'-i',
input.path,
'-frames:v',
'1',
...imageFilterArguments(options.size),
...imageQualityArguments(options.format, options.quality),
output.path,
];
return freezeCommandPlan({
id: `${options.jobId}:thumbnail`,
operation: 'thumbnail',
inputs: [input],
temporaryFiles: [],
args,
outputs: [output],
requiredCapabilities: imageRequirements(options.format),
diagnostics: [],
});
}
export function buildThumbnailSeriesPlan(
options: ThumbnailSeriesOptions
): FFmpegCommandPlan {
const times = resolveThumbnailTimes(options);
const input = createPlannedInput(options.jobId, options.source);
const extension = extensionForImageFormat(options.format);
const outputs = times.map((time, index) =>
createPlannedOutput(options.jobId, {
id: `thumbnail-${index + 1}`,
baseName: options.source.fileName,
operation: 'thumbnail',
suffix: `${String(index + 1).padStart(3, '0')}-${Math.round(time * 1000)}ms`,
extension,
role: 'thumbnail',
})
);
const size = scaleFilter(options.size);
const splitOutputs = times
.map((_, index) => `[thumb-source-${index}]`)
.join('');
const filtergraph = [
`[0:v]split=${times.length}${splitOutputs}`,
...times.map((time, index) => {
const filters = [
`select='gte(t\\,${normalizeSeconds(time)})'`,
'setpts=N/FRAME_RATE/TB',
...(size ? [size] : []),
];
return `[thumb-source-${index}]${filters.join(',')}[thumb-${index}]`;
}),
].join(';');
const outputArguments = outputs.flatMap((output, index) => [
'-map',
`[thumb-${index}]`,
'-frames:v',
'1',
...imageQualityArguments(options.format, options.quality),
output.path,
]);
return freezeCommandPlan({
id: `${options.jobId}:thumbnail-series`,
operation: 'thumbnail-series',
inputs: [input],
temporaryFiles: [],
args: [
'-i',
input.path,
'-filter_complex',
filtergraph,
...outputArguments,
],
outputs,
expectedDurationSeconds: options.durationSeconds,
requiredCapabilities: {
...imageRequirements(options.format),
filters: ['split', 'select', 'setpts', ...(size ? ['scale'] : [])],
},
diagnostics: [
diagnostic(
'single-pass-series',
'info',
'The thumbnail series is decoded in one FFmpeg invocation.'
),
],
});
}
export function buildSceneChangeAnalysisPlan(
options: SceneChangeAnalysisOptions
): FFmpegCommandPlan {
validateSceneAnalysisOptions(options);
const input = createPlannedInput(options.jobId, options.source);
return freezeCommandPlan({
id: `${options.jobId}:scene-change-analysis`,
operation: 'scene-change-analysis',
inputs: [input],
temporaryFiles: [],
args: [
'-i',
input.path,
'-an',
'-vf',
`select='gt(scene\\,${normalizedSceneThreshold(options.threshold)})',showinfo`,
'-frames:v',
String(options.maxFrames),
'-f',
'null',
'-',
],
outputs: [],
expectedDurationSeconds: options.durationSeconds,
requiredCapabilities: {
muxers: ['null'],
filters: ['select', 'showinfo'],
},
diagnostics: [
diagnostic(
'bounded-scene-analysis',
'info',
`Scene analysis stops after ${options.maxFrames} qualifying frames and retains no source media.`
),
],
});
}
export function parseSceneChangeFrames(
logText: string
): readonly SceneChangeFrame[] {
const frames: SceneChangeFrame[] = [];
const seenPts = new Set<number>();
const linePattern =
/^[^\n]*\bn:\s*\d+\s+pts:\s*(-?\d+)\s+pts_time:\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)/gimu;
for (const match of logText.matchAll(linePattern)) {
const pts = Number(match[1]);
const timeSeconds = Number(match[2]);
if (
!Number.isSafeInteger(pts) ||
pts < 0 ||
!Number.isFinite(timeSeconds) ||
timeSeconds < 0 ||
seenPts.has(pts)
) {
continue;
}
seenPts.add(pts);
frames.push({ pts, timeSeconds });
if (frames.length >= MAX_SCENE_CHANGE_FRAMES) {
break;
}
}
return Object.freeze(
frames
.sort((left, right) => left.timeSeconds - right.timeSeconds)
.map((frame) => Object.freeze({ ...frame }))
);
}
export function buildSceneThumbnailSeriesPlan(
options: SceneThumbnailSeriesOptions
): FFmpegCommandPlan {
const frames = validateSceneFrames(
options.frames,
options.durationSeconds,
'Thumbnail series'
);
const input = createPlannedInput(options.jobId, options.source);
const extension = extensionForImageFormat(options.format);
const outputs = frames.map((frame, index) =>
createPlannedOutput(options.jobId, {
id: `thumbnail-${index + 1}`,
baseName: options.source.fileName,
operation: 'thumbnail',
suffix: `${String(index + 1).padStart(3, '0')}-${Math.round(frame.timeSeconds * 1000)}ms`,
extension,
role: 'thumbnail',
})
);
const size = scaleFilter(options.size);
const selected = frames.map((frame) => `eq(pts\\,${frame.pts})`).join('+');
const splitOutputs = frames
.map((_, index) => `[scene-source-${index}]`)
.join('');
const filtergraph = [
`[0:v]select='${selected}',split=${frames.length}${splitOutputs}`,
...frames.map((_, index) => {
const filters = [
`select='eq(n\\,${index})'`,
'setpts=N/FRAME_RATE/TB',
...(size ? [size] : []),
];
return `[scene-source-${index}]${filters.join(',')}[thumb-${index}]`;
}),
].join(';');
const outputArguments = outputs.flatMap((output, index) => [
'-map',
`[thumb-${index}]`,
'-frames:v',
'1',
...imageQualityArguments(options.format, options.quality),
output.path,
]);
return freezeCommandPlan({
id: `${options.jobId}:scene-thumbnail-series`,
operation: 'thumbnail-series',
inputs: [input],
temporaryFiles: [],
args: [
'-i',
input.path,
'-filter_complex',
filtergraph,
...outputArguments,
],
outputs,
expectedDurationSeconds: options.durationSeconds,
requiredCapabilities: {
...imageRequirements(options.format),
filters: ['select', 'split', 'setpts', ...(size ? ['scale'] : [])],
},
diagnostics: [
diagnostic(
'scene-change-selection',
'info',
`${frames.length} frames are selected by exact presentation timestamp from a preceding bounded scene analysis.`
),
],
});
}
export function resolveThumbnailTimes(
options: Pick<
ThumbnailSeriesOptions,
'durationSeconds' | 'timesSeconds' | 'intervalSeconds' | 'count'
>
): readonly number[] {
if (
!Number.isFinite(options.durationSeconds) ||
options.durationSeconds <= 0
) {
throw new RangeError('Source duration must be positive');
}
let times: number[];
if (options.timesSeconds) {
times = [...options.timesSeconds];
} else if (options.intervalSeconds !== undefined) {
if (
!Number.isFinite(options.intervalSeconds) ||
options.intervalSeconds <= 0
) {
throw new RangeError('Thumbnail interval must be positive');
}
if (
Math.ceil(options.durationSeconds / options.intervalSeconds) >
MAX_GENERATED_OUTPUT_FILES
) {
throw new RangeError(
`A thumbnail series can contain at most ${MAX_GENERATED_OUTPUT_FILES} images. ${GENERATED_OUTPUT_LIMIT_EXPLANATION}`
);
}
times = [];
for (
let time = 0;
time < options.durationSeconds;
time += options.intervalSeconds
) {
times.push(time);
}
} else {
const count = options.count ?? 12;
if (
!Number.isSafeInteger(count) ||
count < 1 ||
count > MAX_GENERATED_OUTPUT_FILES
) {
throw new RangeError(
`Thumbnail count must be an integer from 1 to ${MAX_GENERATED_OUTPUT_FILES}. ${GENERATED_OUTPUT_LIMIT_EXPLANATION}`
);
}
times = Array.from(
{ length: count },
(_, index) => (options.durationSeconds * (index + 1)) / (count + 1)
);
}
const unique = [...new Set(times)].sort((left, right) => left - right);
if (
unique.length === 0 ||
unique.length > MAX_GENERATED_OUTPUT_FILES ||
unique.some(
(time) =>
!Number.isFinite(time) || time < 0 || time >= options.durationSeconds
)
) {
throw new RangeError('Thumbnail times must be within the source duration');
}
return Object.freeze(unique);
}
export function buildContactSheetPlan(
options: ContactSheetOptions
): FFmpegCommandPlan {
if (
!Number.isFinite(options.durationSeconds) ||
options.durationSeconds <= 0
) {
throw new RangeError('Contact sheet source duration must be positive');
}
if (
!Number.isSafeInteger(options.frameCount) ||
options.frameCount < 1 ||
options.frameCount > 500 ||
!Number.isSafeInteger(options.rows) ||
!Number.isSafeInteger(options.columns) ||
options.rows < 1 ||
options.columns < 1 ||
options.rows * options.columns < options.frameCount ||
options.rows * options.columns > 500
) {
throw new RangeError(
'Contact sheet grid must contain 1500 cells and fit all frames'
);
}
if (
!Number.isSafeInteger(options.cellWidth) ||
options.cellWidth < 32 ||
options.cellWidth > 4096
) {
throw new RangeError('Contact sheet cell width must be 324096 pixels');
}
const spacing = options.spacing ?? 4;
if (!Number.isSafeInteger(spacing) || spacing < 0 || spacing > 100) {
throw new RangeError('Contact sheet spacing must be 0100 pixels');
}
const background = options.background ?? 'black';
if (!/^(?:[a-z]{3,20}|#[0-9a-f]{6})$/iu.test(background)) {
throw new TypeError('Background must be a named colour or #RRGGBB');
}
const sceneFrames = options.sceneFrames
? validateSceneFrames(
options.sceneFrames,
options.durationSeconds,
'Contact sheet'
)
: undefined;
if (sceneFrames && sceneFrames.length !== options.frameCount) {
throw new RangeError(
'Contact sheet frame count must match the analyzed scene selection'
);
}
const input = createPlannedInput(options.jobId, options.source);
const fontInput = options.font
? createPlannedInput(options.jobId, options.font, 'font')
: undefined;
const output = createPlannedOutput(options.jobId, {
baseName: options.source.fileName,
operation: 'contact-sheet',
extension: extensionForImageFormat(options.format),
role: 'contact-sheet',
});
const fps = options.frameCount / options.durationSeconds;
const filters = [
...(sceneFrames
? [
`select='${sceneFrames
.map((frame) => `eq(pts\\,${frame.pts})`)
.join('+')}'`,
]
: [`fps=${fps.toFixed(8).replace(/0+$/u, '').replace(/\.$/u, '')}`]),
options.maintainAspect === false
? `scale=${options.cellWidth}:${options.cellWidth}`
: `scale=${options.cellWidth}:-2:force_original_aspect_ratio=decrease`,
];
let labelsEnabled = Boolean(options.timestampLabels || options.sourceLabel);
const diagnostics = [];
if (labelsEnabled && (!options.drawtextAvailable || !fontInput)) {
labelsEnabled = false;
diagnostics.push(
diagnostic(
'labels-unavailable',
'warning',
'Timestamp and filename labels were omitted because drawtext or a safe local font is unavailable.'
)
);
}
if (labelsEnabled) {
const text = options.sourceLabel
? `${safePathComponent(options.source.fileName)} %{pts\\:hms}`
: '%{pts\\:hms}';
filters.push(
`drawtext=fontfile='${fontInput?.path ?? ''}':text='${text}':x=8:y=h-th-8:fontcolor=white:box=1:boxcolor=black@0.6`
);
}
if (sceneFrames) {
filters.push('setpts=N/FRAME_RATE/TB');
}
filters.push(
`tile=${options.columns}x${options.rows}:nb_frames=${options.frameCount}:padding=${spacing}:margin=${spacing}:color=${background}`
);
return freezeCommandPlan({
id: `${options.jobId}:contact-sheet`,
operation: 'contact-sheet',
inputs: [input, ...(fontInput ? [fontInput] : [])],
temporaryFiles: [],
args: [
'-i',
input.path,
'-vf',
filters.join(','),
'-frames:v',
'1',
...imageQualityArguments(options.format, options.quality),
output.path,
],
outputs: [output],
expectedDurationSeconds: options.durationSeconds,
requiredCapabilities: {
...imageRequirements(options.format),
filters: [
sceneFrames ? 'select' : 'fps',
'scale',
...(sceneFrames ? ['setpts'] : []),
'tile',
...(labelsEnabled ? ['drawtext'] : []),
],
},
diagnostics: [
...diagnostics,
...(sceneFrames
? [
diagnostic(
'scene-change-selection',
'info',
`${sceneFrames.length} analyzed scene changes populate this contact sheet.`
),
]
: []),
],
});
}
function validateSceneAnalysisOptions(
options: Pick<
SceneChangeAnalysisOptions,
'durationSeconds' | 'threshold' | 'maxFrames'
>
): void {
if (
!Number.isFinite(options.durationSeconds) ||
options.durationSeconds <= 0
) {
throw new RangeError('Scene analysis source duration must be positive');
}
normalizedSceneThreshold(options.threshold);
if (
!Number.isSafeInteger(options.maxFrames) ||
options.maxFrames < 1 ||
options.maxFrames > MAX_SCENE_CHANGE_FRAMES
) {
throw new RangeError(
`Scene analysis frame limit must be an integer from 1 to ${MAX_SCENE_CHANGE_FRAMES}`
);
}
}
function normalizedSceneThreshold(value: number): string {
if (
!Number.isFinite(value) ||
value < MIN_SCENE_CHANGE_THRESHOLD ||
value > MAX_SCENE_CHANGE_THRESHOLD
) {
throw new RangeError(
`Scene threshold must be ${MIN_SCENE_CHANGE_THRESHOLD}${MAX_SCENE_CHANGE_THRESHOLD}`
);
}
return String(Number(value.toFixed(3)));
}
function validateSceneFrames(
frames: readonly SceneChangeFrame[],
durationSeconds: number,
label: string
): readonly SceneChangeFrame[] {
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) {
throw new RangeError(`${label} source duration must be positive`);
}
if (
frames.length < 1 ||
frames.length > MAX_SCENE_CHANGE_FRAMES ||
frames.some(
(frame) =>
!Number.isSafeInteger(frame.pts) ||
frame.pts < 0 ||
!Number.isFinite(frame.timeSeconds) ||
frame.timeSeconds < 0 ||
frame.timeSeconds >= durationSeconds
)
) {
throw new RangeError(
`${label} scene frames must be bounded, finite points inside the source`
);
}
const uniquePts = new Set(frames.map((frame) => frame.pts));
if (uniquePts.size !== frames.length) {
throw new RangeError(`${label} scene frames must have unique timestamps`);
}
return Object.freeze(
[...frames]
.sort((left, right) => left.timeSeconds - right.timeSeconds)
.map((frame) => Object.freeze({ ...frame }))
);
}
function imageFilterArguments(size?: ThumbnailSize): readonly string[] {
const filter = scaleFilter(size);
return filter ? ['-vf', filter] : [];
}
function scaleFilter(size?: ThumbnailSize): string | undefined {
if (!size || (size.width === undefined && size.height === undefined)) {
return undefined;
}
for (const value of [size.width, size.height]) {
if (
value !== undefined &&
(!Number.isSafeInteger(value) || value < 1 || value > 8192)
) {
throw new RangeError(
'Thumbnail dimensions must be integer values from 1 to 8192'
);
}
}
const dimensions = `scale=${size.width ?? -2}:${size.height ?? -2}`;
return size.maintainAspect === false
? dimensions
: `${dimensions}:force_original_aspect_ratio=decrease`;
}
function imageQualityArguments(
format: ImageFormat,
quality?: number
): readonly string[] {
if (quality === undefined) {
return [];
}
if (!Number.isFinite(quality) || quality < 1 || quality > 100) {
throw new RangeError('Image quality must be between 1 and 100');
}
if (format === 'jpeg') {
return ['-q:v', String(Math.max(2, Math.round(31 - quality * 0.29)))];
}
if (format === 'webp') {
return ['-quality', String(Math.round(quality))];
}
return [];
}
function imageRequirements(format: ImageFormat): {
readonly encoders: readonly string[];
readonly muxers: readonly string[];
} {
return format === 'jpeg'
? { encoders: ['mjpeg'], muxers: ['image2'] }
: format === 'png'
? { encoders: ['png'], muxers: ['image2'] }
: { encoders: ['libwebp'], muxers: ['webp'] };
}
function extensionForImageFormat(format: ImageFormat): string {
return format === 'jpeg' ? 'jpg' : format;
}