Files
regex-tools/src/regex/analysis/analysis-runner.ts

820 lines
24 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { WorkerRequestError } from "../execution/WorkerSupervisor";
import {
DEFAULT_REGEX_LIMITS,
utf8ByteLength,
} from "../execution/request-limits";
import { ANALYSIS_LIMITS } from "./analysis-limits";
import type { AnalysisWorkerClient } from "./AnalysisSupervisor";
import type {
AnalysisProgress,
AnalysisRunOptions,
AnalysisRunStatus,
BenchmarkWorkerSample,
GrowthAnalysisRequest,
GrowthAnalysisResult,
GrowthSample,
GrowthStopReason,
RegexBenchmarkRequest,
RegexBenchmarkResult,
RegexRiskFinding,
} from "./analysis.types";
import { summarizeBenchmarkSamples } from "./statistics";
interface Clock {
now(): number;
}
const SYSTEM_CLOCK: Clock = {
now: () => performance.now(),
};
function assertInteger(
value: number,
label: string,
minimum: number,
maximum: number,
): void {
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
throw new RangeError(
`${label} must be an integer from ${minimum.toLocaleString()} to ${maximum.toLocaleString()}.`,
);
}
}
function assertFiniteRange(
value: number,
label: string,
minimum: number,
maximum: number,
): void {
if (!Number.isFinite(value) || value < minimum || value > maximum) {
throw new RangeError(
`${label} must be from ${minimum.toLocaleString()} to ${maximum.toLocaleString()}.`,
);
}
}
export function validateBenchmarkRequest(request: RegexBenchmarkRequest): void {
if (request.flavour !== "ecmascript") {
throw new RangeError("Benchmarking currently supports ECMAScript only.");
}
if (request.pattern.length > DEFAULT_REGEX_LIMITS.patternHardLengthUtf16) {
throw new RangeError("Pattern exceeds the configured hard limit.");
}
if (
utf8ByteLength(request.subject) >
DEFAULT_REGEX_LIMITS.interactiveSubjectHardBytes
) {
throw new RangeError(
"Benchmark subject exceeds the configured hard limit.",
);
}
if (
request.replacement.length >
DEFAULT_REGEX_LIMITS.maximumReplacementTemplateUtf16
) {
throw new RangeError(
"Benchmark replacement exceeds the configured hard limit.",
);
}
assertInteger(
request.maximumMatches,
"Maximum matches",
1,
DEFAULT_REGEX_LIMITS.maximumMatches,
);
assertInteger(
request.settings.warmupIterations,
"Warm-up iterations",
0,
ANALYSIS_LIMITS.maximumWarmupIterations,
);
assertInteger(
request.settings.measuredIterations,
"Measured iterations",
1,
ANALYSIS_LIMITS.maximumMeasuredIterations,
);
assertInteger(
request.settings.sampleTimeoutMs,
"Sample timeout",
ANALYSIS_LIMITS.minimumSampleTimeoutMs,
ANALYSIS_LIMITS.maximumSampleTimeoutMs,
);
assertInteger(
request.settings.maximumWallTimeMs,
"Benchmark wall time",
request.settings.sampleTimeoutMs,
ANALYSIS_LIMITS.maximumAnalysisWallTimeMs,
);
if (
request.settings.warmupIterations +
request.settings.measuredIterations +
1 >
DEFAULT_REGEX_LIMITS.maximumBenchmarkIterations
) {
throw new RangeError(
`Benchmark exceeds the ${DEFAULT_REGEX_LIMITS.maximumBenchmarkIterations.toLocaleString()}-iteration limit.`,
);
}
}
export function validateGrowthRequest(request: GrowthAnalysisRequest): void {
if (request.flavour !== "ecmascript") {
throw new RangeError(
"Dynamic growth analysis currently supports ECMAScript only.",
);
}
if (request.pattern.length > DEFAULT_REGEX_LIMITS.patternHardLengthUtf16) {
throw new RangeError("Pattern exceeds the configured hard limit.");
}
assertInteger(
request.maximumMatches,
"Maximum matches",
1,
DEFAULT_REGEX_LIMITS.maximumMatches,
);
if (request.settings.repeatedFragment.length === 0) {
throw new RangeError("Repeated fragment must not be empty.");
}
if (
request.settings.repeatedFragment.length >
ANALYSIS_LIMITS.maximumGrowthFragmentUtf16
) {
throw new RangeError(
`Repeated fragment exceeds ${ANALYSIS_LIMITS.maximumGrowthFragmentUtf16.toLocaleString()} UTF-16 units.`,
);
}
for (const [label, value] of [
["Growth prefix", request.settings.prefix],
["Growth suffix", request.settings.suffix],
] as const) {
if (value.length > ANALYSIS_LIMITS.maximumGrowthAffixUtf16) {
throw new RangeError(
`${label} exceeds ${ANALYSIS_LIMITS.maximumGrowthAffixUtf16.toLocaleString()} UTF-16 units.`,
);
}
}
assertInteger(
request.settings.startingRepetitions,
"Starting repetitions",
1,
ANALYSIS_LIMITS.maximumGrowthRepetitions,
);
assertInteger(
request.settings.maximumRepetitions,
"Maximum repetitions",
request.settings.startingRepetitions,
ANALYSIS_LIMITS.maximumGrowthRepetitions,
);
assertFiniteRange(request.settings.multiplier, "Growth multiplier", 1.1, 10);
assertInteger(
request.settings.maximumSteps,
"Maximum growth steps",
1,
ANALYSIS_LIMITS.maximumGrowthSteps,
);
assertInteger(
request.settings.sampleTimeoutMs,
"Sample timeout",
ANALYSIS_LIMITS.minimumSampleTimeoutMs,
ANALYSIS_LIMITS.maximumSampleTimeoutMs,
);
assertInteger(
request.settings.maximumWallTimeMs,
"Growth wall time",
request.settings.sampleTimeoutMs,
ANALYSIS_LIMITS.maximumAnalysisWallTimeMs,
);
assertInteger(
request.settings.maximumSubjectBytes,
"Maximum generated subject bytes",
1,
DEFAULT_REGEX_LIMITS.interactiveSubjectHardBytes,
);
assertFiniteRange(
request.settings.normalizedGrowthThreshold,
"Normalized growth threshold",
1.1,
100,
);
}
function errorKind(
error: unknown,
): "timeout" | "cancelled" | "crash" | "error" {
if (error instanceof WorkerRequestError) {
if (error.kind === "timeout") return "timeout";
if (error.kind === "cancelled") return "cancelled";
if (error.kind === "crash") return "crash";
return "error";
}
if (
error &&
typeof error === "object" &&
"kind" in error &&
typeof error.kind === "string"
) {
if (
error.kind === "timeout" ||
error.kind === "cancelled" ||
error.kind === "crash"
) {
return error.kind;
}
}
return "error";
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function aborted(signal: AbortSignal | undefined): boolean {
return signal?.aborted === true;
}
function timeoutWithinDeadline(
configuredTimeoutMs: number,
deadline: number,
clock: Clock,
): number {
return Math.max(
1,
Math.min(configuredTimeoutMs, Math.floor(deadline - clock.now())),
);
}
function benchmarkResult(
request: RegexBenchmarkRequest,
started: number,
clock: Clock,
values: {
readonly status: AnalysisRunStatus;
readonly identity?: RegexBenchmarkResult["identity"];
readonly coldStartMs?: number;
readonly coldSample?: BenchmarkWorkerSample;
readonly warmSamples: readonly BenchmarkWorkerSample[];
readonly completedWarmups: number;
readonly stoppedReason?: string;
readonly warnings: readonly string[];
},
): RegexBenchmarkResult {
return {
status: values.status,
...(values.identity ? { identity: values.identity } : {}),
...(values.coldStartMs === undefined
? {}
: { coldStartMs: values.coldStartMs }),
...(values.coldSample ? { coldSample: values.coldSample } : {}),
warmSamples: values.warmSamples,
warmStatistics: summarizeBenchmarkSamples(values.warmSamples),
completedWarmups: values.completedWarmups,
completedMeasuredIterations: values.warmSamples.length,
requestedSettings: request.settings,
wallTimeMs: Math.max(0, clock.now() - started),
...(values.stoppedReason ? { stoppedReason: values.stoppedReason } : {}),
warnings: values.warnings,
};
}
function progress(options: AnalysisRunOptions, value: AnalysisProgress): void {
options.onProgress?.(value);
}
export async function runRegexBenchmark(
request: RegexBenchmarkRequest,
client: AnalysisWorkerClient,
options: AnalysisRunOptions = {},
clock: Clock = SYSTEM_CLOCK,
): Promise<RegexBenchmarkResult> {
validateBenchmarkRequest(request);
const started = clock.now();
const deadline = started + request.settings.maximumWallTimeMs;
let identity: RegexBenchmarkResult["identity"];
let coldStartMs: number | undefined;
let coldSample: BenchmarkWorkerSample | undefined;
let completedWarmups = 0;
const warmSamples: BenchmarkWorkerSample[] = [];
let requestWasWallLimited = false;
const nextRequestTimeout = () => {
requestWasWallLimited =
deadline - clock.now() < request.settings.sampleTimeoutMs;
return timeoutWithinDeadline(
request.settings.sampleTimeoutMs,
deadline,
clock,
);
};
const warnings = [
"One browser, pattern and subject are not a complete performance characterization.",
"Native browser and WebAssembly engine timings are not directly comparable.",
"Displayed precision does not imply nanosecond measurement accuracy.",
];
const partial = (
status: AnalysisRunStatus,
stoppedReason: string,
): RegexBenchmarkResult =>
benchmarkResult(request, started, clock, {
status,
identity,
coldStartMs,
coldSample,
warmSamples,
completedWarmups,
stoppedReason,
warnings,
});
try {
if (aborted(options.signal)) {
return partial("cancelled", "Benchmark cancelled before it started.");
}
progress(options, {
phase: "starting-worker",
completed: 0,
total: 1,
message: "Starting a fresh ECMAScript analysis worker…",
});
const coldStart = clock.now();
identity = await client.identity(nextRequestTimeout());
coldStartMs = Math.max(0, clock.now() - coldStart);
if (clock.now() >= deadline) {
return partial(
"wall-time-limit",
"Benchmark stopped at its aggregate wall-time limit.",
);
}
progress(options, {
phase: "cold-sample",
completed: 0,
total: 1,
message: "Measuring the first engine use separately…",
});
coldSample = await client.benchmarkSample(request, nextRequestTimeout());
if (!coldSample.accepted) {
return partial(
"error",
coldSample.compileError ??
"The native ECMAScript engine rejected the benchmark pattern.",
);
}
for (let index = 0; index < request.settings.warmupIterations; index += 1) {
if (aborted(options.signal)) {
return partial("cancelled", "Benchmark cancelled during warm-up.");
}
if (clock.now() >= deadline) {
return partial(
"wall-time-limit",
"Benchmark stopped at its aggregate wall-time limit.",
);
}
progress(options, {
phase: "warming-up",
completed: index,
total: request.settings.warmupIterations,
message: `Warm-up ${index + 1} of ${request.settings.warmupIterations}`,
});
const sample = await client.benchmarkSample(
request,
nextRequestTimeout(),
);
if (!sample.accepted) {
return partial(
"error",
sample.compileError ??
"The native ECMAScript engine rejected a warm-up sample.",
);
}
completedWarmups += 1;
}
for (
let index = 0;
index < request.settings.measuredIterations;
index += 1
) {
if (aborted(options.signal)) {
return partial("cancelled", "Benchmark cancelled while measuring.");
}
if (clock.now() >= deadline) {
return partial(
"wall-time-limit",
"Benchmark stopped at its aggregate wall-time limit.",
);
}
progress(options, {
phase: "measuring",
completed: index,
total: request.settings.measuredIterations,
message: `Measured sample ${index + 1} of ${request.settings.measuredIterations}`,
});
const sample = await client.benchmarkSample(
request,
nextRequestTimeout(),
);
if (!sample.accepted) {
return partial(
"error",
sample.compileError ??
"The native ECMAScript engine rejected a measured sample.",
);
}
warmSamples.push(sample);
}
} catch (error) {
const kind = errorKind(error);
if (kind === "timeout") {
if (requestWasWallLimited || clock.now() >= deadline) {
return partial(
"wall-time-limit",
"Benchmark stopped at its aggregate wall-time limit.",
);
}
return partial(
"timeout",
"A benchmark sample timed out; its worker was terminated.",
);
}
if (kind === "cancelled" || aborted(options.signal)) {
return partial(
"cancelled",
"Benchmark cancelled; its worker was terminated.",
);
}
return partial("error", `Benchmark worker failed: ${errorMessage(error)}`);
}
for (const sample of [coldSample, ...warmSamples]) {
if (sample?.matchCollectionTruncated) {
warnings.push(
`Match collection reached the ${request.maximumMatches.toLocaleString()}-match benchmark limit.`,
);
break;
}
}
const skippedReplacement = [coldSample, ...warmSamples].find(
(sample) => sample?.replacementSkippedReason,
)?.replacementSkippedReason;
if (skippedReplacement) {
warnings.push(`Replacement timing skipped: ${skippedReplacement}`);
}
return benchmarkResult(request, started, clock, {
status: "complete",
identity,
coldStartMs,
coldSample,
warmSamples,
completedWarmups,
warnings,
});
}
function generatedSubjectSize(
request: GrowthAnalysisRequest,
repetitions: number,
): { readonly utf16: number; readonly bytes: number } {
return {
utf16:
request.settings.prefix.length +
request.settings.repeatedFragment.length * repetitions +
request.settings.suffix.length,
bytes:
utf8ByteLength(request.settings.prefix) +
utf8ByteLength(request.settings.repeatedFragment) * repetitions +
utf8ByteLength(request.settings.suffix),
};
}
function dynamicFinding(
request: GrowthAnalysisRequest,
rule: "observed-growth" | "observed-timeout",
details: {
readonly confidence: "medium" | "high";
readonly severity: "warning" | "high";
readonly explanation: string;
readonly exampleRisk: string;
},
): RegexRiskFinding {
return {
id: `risk-${rule}-0-${request.pattern.length}`,
flavour: "ecmascript",
range: { startUtf16: 0, endUtf16: request.pattern.length },
rule,
title:
rule === "observed-timeout"
? "Observed timeout under selected limits"
: "Observed disproportionate growth",
explanation: details.explanation,
exampleRisk: details.exampleRisk,
confidence: details.confidence,
severity: details.severity,
limitations:
"This observation applies only to the generated subjects, browser runtime and limits shown in this result.",
suggestedInvestigation:
"Repeat with representative production boundaries, retain strict worker limits, and simplify ambiguous repetition where possible.",
evidence: "dynamically-observed",
};
}
function growthResult(
request: GrowthAnalysisRequest,
started: number,
clock: Clock,
values: {
readonly status: AnalysisRunStatus;
readonly identity?: GrowthAnalysisResult["identity"];
readonly samples: readonly GrowthSample[];
readonly stopReason: GrowthStopReason;
readonly stoppedReason: string;
readonly dynamicFindings: readonly RegexRiskFinding[];
},
): GrowthAnalysisResult {
return {
status: values.status,
...(values.identity ? { identity: values.identity } : {}),
samples: values.samples,
stopReason: values.stopReason,
stoppedReason: values.stoppedReason,
wallTimeMs: Math.max(0, clock.now() - started),
dynamicFindings: values.dynamicFindings,
settings: request.settings,
};
}
export async function runGrowthAnalysis(
request: GrowthAnalysisRequest,
client: AnalysisWorkerClient,
options: AnalysisRunOptions = {},
clock: Clock = SYSTEM_CLOCK,
): Promise<GrowthAnalysisResult> {
validateGrowthRequest(request);
const started = clock.now();
const deadline = started + request.settings.maximumWallTimeMs;
const samples: GrowthSample[] = [];
const dynamicFindings: RegexRiskFinding[] = [];
let identity: GrowthAnalysisResult["identity"];
let currentRepetitions = request.settings.startingRepetitions;
let requestWasWallLimited = false;
const nextRequestTimeout = () => {
requestWasWallLimited =
deadline - clock.now() < request.settings.sampleTimeoutMs;
return timeoutWithinDeadline(
request.settings.sampleTimeoutMs,
deadline,
clock,
);
};
const finish = (
status: AnalysisRunStatus,
stopReason: GrowthStopReason,
stoppedReason: string,
) =>
growthResult(request, started, clock, {
status,
identity,
samples,
stopReason,
stoppedReason,
dynamicFindings,
});
try {
if (aborted(options.signal)) {
return finish(
"cancelled",
"cancelled",
"Growth analysis cancelled before it started.",
);
}
identity = await client.identity(nextRequestTimeout());
while (samples.length < request.settings.maximumSteps) {
if (aborted(options.signal)) {
return finish(
"cancelled",
"cancelled",
"Growth analysis cancelled; its worker was terminated.",
);
}
if (clock.now() >= deadline) {
return finish(
"wall-time-limit",
"wall-time-limit",
"Growth analysis stopped at its aggregate wall-time limit.",
);
}
if (currentRepetitions > request.settings.maximumRepetitions) {
return finish(
"complete",
"maximum-repetitions",
"Growth analysis reached the configured repetition bound.",
);
}
const size = generatedSubjectSize(request, currentRepetitions);
if (
!Number.isSafeInteger(size.bytes) ||
!Number.isSafeInteger(size.utf16) ||
size.bytes > request.settings.maximumSubjectBytes
) {
return finish(
"complete",
"maximum-subject-size",
"Growth analysis stopped before exceeding the generated-subject byte limit.",
);
}
progress(options, {
phase: "growth-probe",
completed: samples.length,
total: request.settings.maximumSteps,
message: `Probing ${currentRepetitions.toLocaleString()} fragment repetitions…`,
});
const subject =
request.settings.prefix +
request.settings.repeatedFragment.repeat(currentRepetitions) +
request.settings.suffix;
let workerSample;
try {
workerSample = await client.growthProbe(
{
flavour: "ecmascript",
pattern: request.pattern,
flags: request.flags,
subject,
scanAll: request.scanAll,
maximumMatches: request.maximumMatches,
},
nextRequestTimeout(),
);
} catch (error) {
const kind = errorKind(error);
if (kind === "timeout") {
if (requestWasWallLimited || clock.now() >= deadline) {
return finish(
"wall-time-limit",
"wall-time-limit",
"Growth analysis stopped at its aggregate wall-time limit.",
);
}
samples.push({
repetitions: currentRepetitions,
inputUtf16: size.utf16,
inputBytes: size.bytes,
status: "timeout",
message: "Worker timed out and was terminated.",
});
dynamicFindings.push(
dynamicFinding(request, "observed-timeout", {
confidence: "high",
severity: "high",
explanation: `Execution did not finish within ${request.settings.sampleTimeoutMs.toLocaleString()} ms at ${currentRepetitions.toLocaleString()} generated repetitions.`,
exampleRisk:
"The selected generated near-miss exhausted the configured per-sample wall time.",
}),
);
return finish(
"timeout",
"timeout",
"Observed timeout under selected limits; the worker was terminated.",
);
}
if (kind === "cancelled" || aborted(options.signal)) {
return finish(
"cancelled",
"cancelled",
"Growth analysis cancelled; its worker was terminated.",
);
}
samples.push({
repetitions: currentRepetitions,
inputUtf16: size.utf16,
inputBytes: size.bytes,
status: "crash",
message: errorMessage(error),
});
return finish(
"error",
"crash",
`Growth worker failed: ${errorMessage(error)}`,
);
}
if (!workerSample.accepted) {
samples.push({
repetitions: currentRepetitions,
inputUtf16: size.utf16,
inputBytes: size.bytes,
status: "compile-error",
message: workerSample.compileError,
});
return finish(
"error",
"compile-error",
workerSample.compileError ??
"The native ECMAScript engine rejected the pattern.",
);
}
const previous = samples.at(-1);
const executionMs = workerSample.executionMs ?? 0;
let normalizedGrowth: number | undefined;
if (
previous?.status === "complete" &&
previous.executionMs !== undefined &&
previous.executionMs >= 0.25 &&
executionMs >= 1 &&
previous.inputBytes > 0
) {
const inputRatio = size.bytes / previous.inputBytes;
const timeRatio = executionMs / previous.executionMs;
if (inputRatio > 0) normalizedGrowth = timeRatio / inputRatio;
}
samples.push({
repetitions: currentRepetitions,
inputUtf16: size.utf16,
inputBytes: size.bytes,
status: "complete",
executionMs,
matched: workerSample.matched,
matchCount: workerSample.matchCount,
matchCollectionTruncated: workerSample.matchCollectionTruncated,
...(normalizedGrowth === undefined ? {} : { normalizedGrowth }),
});
if (
normalizedGrowth !== undefined &&
normalizedGrowth >= request.settings.normalizedGrowthThreshold
) {
dynamicFindings.push(
dynamicFinding(request, "observed-growth", {
confidence: "medium",
severity: "warning",
explanation: `Observed time growth was ${normalizedGrowth.toFixed(2)}× the input growth between the last two generated samples.`,
exampleRisk:
"Execution time increased disproportionately for the selected generated subject family.",
}),
);
return finish(
"complete",
"growth-threshold",
"Growth analysis stopped at the configured normalized-growth threshold.",
);
}
if (currentRepetitions >= request.settings.maximumRepetitions) {
return finish(
"complete",
"maximum-repetitions",
"Growth analysis reached the configured repetition bound.",
);
}
const next = Math.min(
request.settings.maximumRepetitions,
Math.max(
currentRepetitions + 1,
Math.ceil(currentRepetitions * request.settings.multiplier),
),
);
currentRepetitions = next;
}
} catch (error) {
const kind = errorKind(error);
if (kind === "timeout") {
if (requestWasWallLimited || clock.now() >= deadline) {
return finish(
"wall-time-limit",
"wall-time-limit",
"Growth analysis stopped at its aggregate wall-time limit.",
);
}
return finish(
"timeout",
"timeout",
"The analysis worker timed out while starting and was terminated.",
);
}
if (kind === "cancelled" || aborted(options.signal)) {
return finish(
"cancelled",
"cancelled",
"Growth analysis cancelled; its worker was terminated.",
);
}
return finish(
"error",
"crash",
`Growth worker failed: ${errorMessage(error)}`,
);
}
return finish(
"complete",
"maximum-steps",
"Growth analysis reached the configured step limit.",
);
}