av-tools/tests/browser/performance.spec.ts

/// <reference types="node" />

import { expect, test, type Browser, type Page } from '@playwright/test';
import { execFileSync } from 'node:child_process';
import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PRIMARY_FIXTURE } from './browser-helpers';

interface BenchmarkResult {
  readonly mode: 'single-thread' | 'multithread';
  readonly source: 'small' | 'medium';
  readonly sourceDescription: string;
  readonly cacheCondition: string;
  readonly initMs: number;
  readonly probeMs: number;
  readonly conversionMs: number;
  readonly outputReadMs: number;
  readonly cleanupMs: number;
  readonly exportEndToEndMs: number;
  readonly outputBytes: number;
  readonly hardwareConcurrency: number;
  readonly crossOriginIsolated: boolean;
  readonly inputBytes: number;
  readonly usedJsHeapBytes?: number;
}

test.describe('opt-in pinned-core performance record', () => {
  test.skip(
    process.env.AV_BENCHMARK !== '1',
    'Run with AV_BENCHMARK=1 through npm run benchmark:browser.'
  );

  test('records comparable fresh-context ST and MT end-to-end timings', async ({
    browser,
  }, testInfo) => {
    test.setTimeout(600_000);
    const medium = createMediumFixture();
    const results: BenchmarkResult[] = [];
    try {
      results.push(
        await runMode(browser, 'single-thread', PRIMARY_FIXTURE, 'small')
      );
      results.push(
        await runMode(browser, 'multithread', PRIMARY_FIXTURE, 'small')
      );
      results.push(
        await runMode(browser, 'single-thread', medium.filePath, 'medium')
      );
      results.push(
        await runMode(browser, 'multithread', medium.filePath, 'medium')
      );
      await testInfo.attach('av-tools-browser-benchmark.json', {
        body: Buffer.from(`${JSON.stringify(results, null, 2)}\n`),
        contentType: 'application/json',
      });
      process.stdout.write(`AV_BENCHMARK_RESULT ${JSON.stringify(results)}\n`);
    } finally {
      medium.dispose();
    }
  });
});

async function runMode(
  browser: Browser,
  mode: BenchmarkResult['mode'],
  fixturePath: string,
  source: BenchmarkResult['source']
): Promise<BenchmarkResult> {
  const context = await browser.newContext();
  const page = await context.newPage();
  try {
    page.on('dialog', (dialog) => void dialog.accept());
    await page.addInitScript(() => {
      Reflect.deleteProperty(globalThis, 'showSaveFilePicker');
    });
    await page.goto('http://127.0.0.1:4173/');
    const environment = await page.evaluate(() => ({
      hardwareConcurrency: navigator.hardwareConcurrency,
      crossOriginIsolated: globalThis.crossOriginIsolated,
    }));
    if (mode === 'multithread') {
      expect(environment.crossOriginIsolated).toBe(true);
    }

    await page.getByText('Engine & browser').click();
    await page
      .getByLabel('Engine mode')
      .selectOption(
        mode === 'single-thread' ? 'force-single-thread' : 'prefer-multithread'
      );
    const initStarted = await monotonicNow(page);
    await page.getByRole('button', { name: 'Initialize engine' }).click();
    await expect(
      page.locator('.engine-pill').filter({
        hasText:
          mode === 'single-thread'
            ? 'Single-threaded compatibility mode'
            : 'Multithreaded',
      })
    ).toBeVisible({ timeout: 120_000 });
    const initMs = (await monotonicNow(page)) - initStarted;

    const probeStarted = await monotonicNow(page);
    await mediaInput(page).setInputFiles(fixturePath);
    await page.getByRole('button', { name: /^Inspect details for /u }).click();
    await expect(page.locator('.source-summary .phase--ready')).toHaveText(
      'Details ready',
      { timeout: 120_000 }
    );
    const probeMs = (await monotonicNow(page)) - probeStarted;

    const exportStarted = await monotonicNow(page);
    await page.getByRole('button', { name: 'Convert', exact: true }).click();
    const result = page.locator('.result-card').last();
    await expect(result).toContainText(
      /created · verified|completed locally/u,
      {
        timeout: 180_000,
      }
    );
    const exportEndToEndMs = (await monotonicNow(page)) - exportStarted;
    const reportDownload = page.waitForEvent('download');
    await page
      .getByRole('button', { name: 'Export validation report' })
      .click();
    const reportPath = await (await reportDownload).path();
    expect(reportPath).not.toBeNull();
    const report = JSON.parse(
      readFileSync(reportPath as string, 'utf8')
    ) as PerformanceReport;
    const outputBytes = await page
      .locator('.quick-preview video')
      .evaluate(async (element) => {
        const response = await fetch((element as HTMLVideoElement).src);
        return (await response.arrayBuffer()).byteLength;
      });
    const usedJsHeapBytes = await page.evaluate(() => {
      const memory = (
        performance as Performance & {
          memory?: { usedJSHeapSize?: number };
        }
      ).memory;
      return memory?.usedJSHeapSize;
    });

    const benchmarkSample: BenchmarkResult = {
      mode,
      source,
      sourceDescription:
        source === 'small'
          ? '2 s, 160×90 H.264/AAC synthetic pattern'
          : '6 s, 640×360 H.264/AAC synthetic loop',
      cacheCondition:
        'fresh browser context; browser HTTP cache may be warm after the first run',
      initMs: rounded(initMs),
      probeMs: rounded(probeMs),
      conversionMs: rounded(report.execution.conversionMilliseconds),
      outputReadMs: rounded(report.execution.outputReadMilliseconds),
      cleanupMs: rounded(report.execution.cleanupMilliseconds),
      exportEndToEndMs: rounded(exportEndToEndMs),
      outputBytes,
      hardwareConcurrency: environment.hardwareConcurrency,
      crossOriginIsolated: environment.crossOriginIsolated,
      inputBytes: statSync(fixturePath).size,
      ...(usedJsHeapBytes === undefined ? {} : { usedJsHeapBytes }),
    };
    process.stdout.write(
      `AV_BENCHMARK_SAMPLE ${JSON.stringify(benchmarkSample)}\n`
    );
    return benchmarkSample;
  } finally {
    await context.close();
  }
}

interface PerformanceReport {
  readonly execution: {
    readonly conversionMilliseconds: number;
    readonly outputReadMilliseconds: number;
    readonly cleanupMilliseconds: number;
  };
}

function createMediumFixture(): {
  readonly filePath: string;
  readonly dispose: () => void;
} {
  const directory = mkdtempSync(join(tmpdir(), 'av-tools-benchmark-'));
  const filePath = join(directory, 'medium-pattern.mp4');
  try {
    execFileSync(
      'ffmpeg',
      [
        '-hide_banner',
        '-loglevel',
        'error',
        '-y',
        '-stream_loop',
        '2',
        '-i',
        PRIMARY_FIXTURE,
        '-t',
        '6',
        '-vf',
        'scale=640:360',
        '-r',
        '24',
        '-c:v',
        'libx264',
        '-preset',
        'ultrafast',
        '-crf',
        '30',
        '-c:a',
        'aac',
        '-b:a',
        '64k',
        filePath,
      ],
      { stdio: 'pipe' }
    );
  } catch (error) {
    rmSync(directory, { recursive: true, force: true });
    throw error;
  }
  return {
    filePath,
    dispose: () => rmSync(directory, { recursive: true, force: true }),
  };
}

function mediaInput(page: Page) {
  return page.locator('input[type="file"][accept*="audio"]').first();
}

async function monotonicNow(page: Page): Promise<number> {
  return page.evaluate(() => performance.now());
}

function rounded(value: number): number {
  return Math.round(value * 10) / 10;
}