perf: bound OneNote TOC hierarchy building

This commit is contained in:
2026-07-22 19:54:23 +02:00
parent b6bb75f976
commit ebcaf436cf
4 changed files with 505 additions and 83 deletions

View File

@@ -194,9 +194,10 @@ async function buildOpenedPackage(
}); });
} }
const treeResult = buildOneNotePackageTree( const treeResult = await buildOneNotePackageTree(
extraction.extractedEntries, extraction.extractedEntries,
packagedSections packagedSections,
options
); );
diagnostics.push(...treeResult.diagnostics); diagnostics.push(...treeResult.diagnostics);

View File

@@ -1,6 +1,29 @@
import { describe, expect, it } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest';
import { resolveTocChildPath } from './toc-hierarchy.js'; import type { OneNotePackagedSectionDto } from '../model/dto.js';
import type {
OneNoteTableOfContents,
OneNoteTableOfContentsEntry,
} from '../parser/parse-toc.js';
import {
DESKTOP_REVISION_STORE_FORMAT,
ONE_SECTION_FILE_TYPE,
} from '../parser/detect-format.js';
import type { OnePkgError } from './errors.js';
import {
buildOneNotePackageTree,
resolveTocChildPath,
} from './toc-hierarchy.js';
import type { ExtractedCabFile } from './types.js';
const parseTocMock = vi.hoisted(() => vi.fn());
vi.mock('../parser/parse-toc.js', async (importOriginal) => ({
...(await importOriginal<typeof import('../parser/parse-toc.js')>()),
parseOneNoteTableOfContents: parseTocMock,
}));
beforeEach(() => parseTocMock.mockReset());
describe('ONEPKG TOC child paths', () => { describe('ONEPKG TOC child paths', () => {
it('joins a single NFC-normalized child name within its section group', () => { it('joins a single NFC-normalized child name within its section group', () => {
@@ -18,3 +41,244 @@ describe('ONEPKG TOC child paths', () => {
} }
); );
}); });
describe('ONEPKG TOC hierarchy budgets and indexes', () => {
it('resolves identity fallbacks in authored order with near-linear section lookups', async () => {
const count = 400;
const pathReads = { count: 0 };
const entries: OneNoteTableOfContentsEntry[] = [];
const extracted: ExtractedCabFile[] = [tocFile()];
const sections: OneNotePackagedSectionDto[] = [];
for (let index = 0; index < count; index += 1) {
const identity = indexedGuid(index + 1);
const path = `Section-${index.toString().padStart(4, '0')}.one`;
extracted.push(sectionFile(path, identity, index + 1));
sections.push(sectionWithObservedPath(path, pathReads));
entries.push({
filename: `Alias-${index.toString().padStart(4, '0')}.one`,
fileIdentity: identity,
orderingId: index,
});
}
parseTocMock.mockReturnValue(toc(entries));
let yields = 0;
const result = await buildOneNotePackageTree(extracted, sections, {
limits: { maxTocOperations: count * 8 + 100 },
cancellation: {
yield: () => {
yields += 1;
},
},
});
expect(result.tree.nodes).toHaveLength(count);
expect(
result.tree.nodes.map((node) =>
node.kind === 'section' ? node.sectionId : `group:${node.id}`
)
).toEqual(sections.map(({ id }) => id));
expect(pathReads.count).toBeLessThanOrEqual(count * 5);
expect(yields).toBeGreaterThan(0);
});
it('keeps ambiguous identities unresolved and fallback sections sorted', async () => {
const identity = indexedGuid(42);
parseTocMock.mockReturnValue(
toc([{ filename: 'Alias.one', fileIdentity: identity, orderingId: 1 }])
);
const sections = [section('Z.one'), section('A.one')];
const result = await buildOneNotePackageTree(
[
tocFile(),
sectionFile('Z.one', identity, 1),
sectionFile('A.one', identity, 2),
],
sections
);
expect(
result.tree.nodes.map((node) =>
node.kind === 'section' ? node.sectionId : `group:${node.id}`
)
).toEqual(['A.one', 'Z.one']);
expect(result.diagnostics.map(({ code }) => code)).toEqual(
expect.arrayContaining([
'onetoc2-entry-unresolved',
'onetoc2-unlisted-sections',
])
);
expect(result.diagnostics).not.toEqual(
expect.arrayContaining([
expect.objectContaining({
code: 'onetoc2-section-filename-mismatch',
}),
])
);
});
it('caps adversarial unresolved-entry diagnostics with an explicit marker', async () => {
parseTocMock.mockReturnValue(
toc(
Array.from({ length: 50 }, (_, index) => ({
filename: `Missing-${index}.one`,
fileIdentity: indexedGuid(index + 1),
orderingId: index,
}))
)
);
const result = await buildOneNotePackageTree([tocFile()], [], {
limits: { maxTocDiagnostics: 5, maxTocOperations: 1_000 },
});
expect(result.diagnostics).toHaveLength(5);
expect(result.diagnostics.slice(0, 4).map(({ code }) => code)).toEqual([
'onetoc2-entry-unresolved',
'onetoc2-entry-unresolved',
'onetoc2-entry-unresolved',
'onetoc2-entry-unresolved',
]);
expect(result.diagnostics[4]?.code).toBe('onetoc2-diagnostic-limit');
});
it('fails closed at the hierarchy operation limit', async () => {
parseTocMock.mockReturnValue(
toc(
Array.from({ length: 20 }, (_, index) => ({
filename: `Missing-${index}.one`,
fileIdentity: indexedGuid(index + 1),
orderingId: index,
}))
)
);
await expect(
buildOneNotePackageTree([tocFile()], [], {
limits: { maxTocOperations: 10 },
})
).rejects.toEqual(
expect.objectContaining<Partial<OnePkgError>>({
diagnostic: expect.objectContaining({
code: 'onetoc2-operation-limit',
}),
})
);
});
it('yields and observes cancellation during a large hierarchy build', async () => {
parseTocMock.mockReturnValue(
toc(
Array.from({ length: 1_000 }, (_, index) => ({
filename: `Missing-${index}.one`,
fileIdentity: indexedGuid(index + 1),
orderingId: index,
}))
)
);
const controller = new AbortController();
let yields = 0;
await expect(
buildOneNotePackageTree([tocFile()], [], {
cancellation: {
signal: controller.signal,
yield: () => {
yields += 1;
controller.abort();
},
},
})
).rejects.toEqual(
expect.objectContaining<Partial<OnePkgError>>({
diagnostic: expect.objectContaining({ code: 'onepkg-cancelled' }),
})
);
expect(yields).toBe(1);
});
});
function toc(entries: OneNoteTableOfContentsEntry[]): OneNoteTableOfContents {
return {
sourceName: 'Open Notebook.onetoc2',
fileIdentity: indexedGuid(0),
entries,
diagnostics: [],
};
}
function tocFile(): ExtractedCabFile {
return extractedFile('Open Notebook.onetoc2', Uint8Array.of(1), 0);
}
function sectionFile(
path: string,
identity: string,
index: number
): ExtractedCabFile {
const bytes = new Uint8Array(64);
writeGuid(bytes, 0, ONE_SECTION_FILE_TYPE);
writeGuid(bytes, 16, identity);
writeGuid(bytes, 48, DESKTOP_REVISION_STORE_FORMAT);
return extractedFile(path, bytes, index);
}
function extractedFile(
path: string,
bytes: Uint8Array,
index: number
): ExtractedCabFile {
return {
index,
path,
normalizedPath: path,
folderIndex: 0,
uncompressedOffset: 0,
uncompressedSize: bytes.byteLength,
attributes: 0,
compression: { kind: 'none', raw: 0, label: 'None' },
bytes,
};
}
function section(path: string): OneNotePackagedSectionDto {
return {
id: path,
path,
displayName: path.slice(0, -4),
parseStatus: 'not-requested',
diagnostics: [],
};
}
function sectionWithObservedPath(
path: string,
reads: { count: number }
): OneNotePackagedSectionDto {
const result = section(path);
Object.defineProperty(result, 'path', {
enumerable: true,
get: () => {
reads.count += 1;
return path;
},
});
return result;
}
function indexedGuid(index: number): string {
const plain = index.toString(16).toUpperCase().padStart(32, '0');
return `{${plain.slice(0, 8)}-${plain.slice(8, 12)}-${plain.slice(12, 16)}-${plain.slice(16, 20)}-${plain.slice(20)}}`;
}
function writeGuid(target: Uint8Array, offset: number, guid: string): void {
const plain = guid.replace(/[{}-]/gu, '');
const canonical = Array.from({ length: 16 }, (_, index) =>
Number.parseInt(plain.slice(index * 2, index * 2 + 2), 16)
);
const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15];
for (let index = 0; index < order.length; index += 1) {
target[offset + order[index]!] = canonical[index]!;
}
}

View File

@@ -16,9 +16,16 @@ import {
type OneNoteTableOfContents, type OneNoteTableOfContents,
type OneNoteTableOfContentsEntry, type OneNoteTableOfContentsEntry,
} from '../parser/parse-toc.js'; } from '../parser/parse-toc.js';
import { checkCancellation, yieldForCancellation } from './cancellation.js';
import { fail, OnePkgError, warning } from './errors.js'; import { fail, OnePkgError, warning } from './errors.js';
import type { ExtractedCabFile } from './types.js'; import {
import { DEFAULT_ONEPKG_LIMITS } from './types.js'; DEFAULT_ONEPKG_LIMITS,
resolveLimits,
type CabOpenOptions,
type ExtractedCabFile,
type OnePkgCancellation,
type OnePkgLimits,
} from './types.js';
interface ParsedTocEntry { interface ParsedTocEntry {
extracted: ExtractedCabFile; extracted: ExtractedCabFile;
@@ -35,18 +42,24 @@ export interface OneNotePackageTreeResult {
* Interpret package TOCs and join them to extracted sections without ever * Interpret package TOCs and join them to extracted sections without ever
* materializing an archive path on the host filesystem. * materializing an archive path on the host filesystem.
*/ */
export function buildOneNotePackageTree( export async function buildOneNotePackageTree(
extractedEntries: ExtractedCabFile[], extractedEntries: ExtractedCabFile[],
sections: OneNotePackagedSectionDto[] sections: OneNotePackagedSectionDto[],
): OneNotePackageTreeResult { options: CabOpenOptions = {}
const diagnostics: ParserDiagnostic[] = []; ): Promise<OneNotePackageTreeResult> {
const limits = resolveLimits(options.limits);
const budget = new TocBuildBudget(limits, options.cancellation);
const diagnostics = new CappedTocDiagnostics(limits.maxTocDiagnostics);
const parsedTocs: ParsedTocEntry[] = []; const parsedTocs: ParsedTocEntry[] = [];
for (const extracted of extractedEntries) { for (const extracted of extractedEntries) {
const checkpoint = budget.charge();
if (checkpoint) await checkpoint;
if (!extracted.normalizedPath.toLowerCase().endsWith('.onetoc2')) continue; if (!extracted.normalizedPath.toLowerCase().endsWith('.onetoc2')) continue;
try { try {
const toc = parseOneNoteTableOfContents(extracted.bytes, { const toc = parseOneNoteTableOfContents(extracted.bytes, {
sourceName: extracted.normalizedPath, sourceName: extracted.normalizedPath,
limits: { maxDiagnostics: limits.maxTocDiagnostics },
}); });
parsedTocs.push({ parsedTocs.push({
extracted, extracted,
@@ -54,8 +67,9 @@ export function buildOneNotePackageTree(
directory: parentPath(extracted.normalizedPath), directory: parentPath(extracted.normalizedPath),
}); });
for (const diagnostic of toc.diagnostics) { for (const diagnostic of toc.diagnostics) {
if (diagnostics.length >= DEFAULT_ONEPKG_LIMITS.maxFiles) break; const diagnosticCheckpoint = budget.charge();
diagnostics.push({ if (diagnosticCheckpoint) await diagnosticCheckpoint;
diagnostics.add({
...diagnostic, ...diagnostic,
structure: diagnostic.structure structure: diagnostic.structure
? `${extracted.normalizedPath} · ${diagnostic.structure}` ? `${extracted.normalizedPath} · ${diagnostic.structure}`
@@ -63,29 +77,63 @@ export function buildOneNotePackageTree(
}); });
} }
} catch (error) { } catch (error) {
diagnostics.push(tocFailureDiagnostic(error, extracted.normalizedPath)); if (isFatalTocBuildError(error)) throw error;
diagnostics.add(tocFailureDiagnostic(error, extracted.normalizedPath));
} }
} }
const root = selectRootToc(parsedTocs, diagnostics); const root = await selectRootToc(parsedTocs, diagnostics, budget);
const sectionByPath = new Map( const sectionByPath = new Map<string, OneNotePackagedSectionDto>();
sections.map((section) => [pathKey(section.path), section]) for (const section of sections) {
); const checkpoint = budget.charge();
const extractedSectionByPath = new Map( if (checkpoint) await checkpoint;
extractedEntries sectionByPath.set(pathKey(section.path), section);
.filter((entry) => entry.normalizedPath.toLowerCase().endsWith('.one')) }
.map((entry) => [pathKey(entry.normalizedPath), entry]) const extractedSectionByPath = new Map<string, ExtractedCabFile>();
); for (const entry of extractedEntries) {
const checkpoint = budget.charge();
if (checkpoint) await checkpoint;
if (!entry.normalizedPath.toLowerCase().endsWith('.one')) continue;
extractedSectionByPath.set(pathKey(entry.normalizedPath), entry);
}
const sectionIdentityByPath = new Map<string, string>(); const sectionIdentityByPath = new Map<string, string>();
for (const [key, entry] of extractedSectionByPath) { for (const [key, entry] of extractedSectionByPath) {
const checkpoint = budget.charge();
if (checkpoint) await checkpoint;
const identity = readSectionIdentity(entry.bytes); const identity = readSectionIdentity(entry.bytes);
if (identity) sectionIdentityByPath.set(key, identity); if (identity) sectionIdentityByPath.set(key, identity);
} }
const sectionByDirectoryAndIdentity = new Map<
string,
Map<string, IdentitySectionMatch>
>();
for (const section of sections) {
const checkpoint = budget.charge();
if (checkpoint) await checkpoint;
const sectionPathKey = pathKey(section.path);
const identity = sectionIdentityByPath.get(sectionPathKey);
if (!identity) continue;
const directoryKey = pathKey(parentPath(section.path));
let byIdentity = sectionByDirectoryAndIdentity.get(directoryKey);
if (!byIdentity) {
byIdentity = new Map();
sectionByDirectoryAndIdentity.set(directoryKey, byIdentity);
}
const existing = byIdentity.get(identity);
byIdentity.set(
identity,
existing
? { section: existing.section, ambiguous: true }
: { section, ambiguous: false }
);
}
const tocByDirectory = new Map<string, ParsedTocEntry>(); const tocByDirectory = new Map<string, ParsedTocEntry>();
for (const parsed of parsedTocs) { for (const parsed of parsedTocs) {
const checkpoint = budget.charge();
if (checkpoint) await checkpoint;
const key = pathKey(parsed.directory); const key = pathKey(parsed.directory);
if (tocByDirectory.has(key)) { if (tocByDirectory.has(key)) {
diagnostics.push( diagnostics.add(
warning( warning(
'onetoc2-directory-ambiguous', 'onetoc2-directory-ambiguous',
`More than one readable .onetoc2 exists in ${parsed.directory || 'the package root'}; the first one is used.`, `More than one readable .onetoc2 exists in ${parsed.directory || 'the package root'}; the first one is used.`,
@@ -100,28 +148,36 @@ export function buildOneNotePackageTree(
const usedSectionIds = new Set<string>(); const usedSectionIds = new Set<string>();
let nodes: OneNoteNotebookNodeDto[] = []; let nodes: OneNoteNotebookNodeDto[] = [];
if (root) { if (root) {
nodes = buildTocNodes({ nodes = await buildTocNodes({
parsed: root, parsed: root,
tocByDirectory, tocByDirectory,
sectionByPath, sectionByPath,
sectionIdentityByPath, sectionIdentityByPath,
sections, sectionByDirectoryAndIdentity,
usedSectionIds, usedSectionIds,
diagnostics, diagnostics,
limits,
budget,
activeTocs: new Set(), activeTocs: new Set(),
usedTocs: new Set(), usedTocs: new Set(),
depth: 0, depth: 0,
}); });
} }
const unlisted = sections const unlisted: OneNotePackagedSectionDto[] = [];
.filter( for (const section of sections) {
(section) => const checkpoint = budget.charge();
!usedSectionIds.has(section.id) && !isRecycleBinPath(section.path) if (checkpoint) await checkpoint;
) if (!usedSectionIds.has(section.id) && !isRecycleBinPath(section.path)) {
.sort((left, right) => left.path.localeCompare(right.path)); unlisted.push(section);
}
}
unlisted.sort((left, right) => {
budget.chargeWithoutYield();
return left.path.localeCompare(right.path);
});
if (unlisted.length > 0 && root) { if (unlisted.length > 0 && root) {
diagnostics.push( diagnostics.add(
warning( warning(
'onetoc2-unlisted-sections', 'onetoc2-unlisted-sections',
`${unlisted.length} package section${unlisted.length === 1 ? '' : 's'} were not reachable from the parsed TOC and were appended to navigation.`, `${unlisted.length} package section${unlisted.length === 1 ? '' : 's'} were not reachable from the parsed TOC and were appended to navigation.`,
@@ -130,6 +186,7 @@ export function buildOneNotePackageTree(
); );
} }
nodes.push(...unlisted.map(fallbackSectionNode)); nodes.push(...unlisted.map(fallbackSectionNode));
await budget.finish();
return { return {
tree: { tree: {
@@ -138,28 +195,38 @@ export function buildOneNotePackageTree(
color: root?.toc.color, color: root?.toc.color,
nodes, nodes,
}, },
diagnostics, diagnostics: diagnostics.values(),
}; };
} }
interface IdentitySectionMatch {
section: OneNotePackagedSectionDto;
ambiguous: boolean;
}
interface BuildTocNodesContext { interface BuildTocNodesContext {
parsed: ParsedTocEntry; parsed: ParsedTocEntry;
tocByDirectory: ReadonlyMap<string, ParsedTocEntry>; tocByDirectory: ReadonlyMap<string, ParsedTocEntry>;
sectionByPath: ReadonlyMap<string, OneNotePackagedSectionDto>; sectionByPath: ReadonlyMap<string, OneNotePackagedSectionDto>;
sectionIdentityByPath: ReadonlyMap<string, string>; sectionIdentityByPath: ReadonlyMap<string, string>;
sections: OneNotePackagedSectionDto[]; sectionByDirectoryAndIdentity: ReadonlyMap<
string,
ReadonlyMap<string, IdentitySectionMatch>
>;
usedSectionIds: Set<string>; usedSectionIds: Set<string>;
diagnostics: ParserDiagnostic[]; diagnostics: CappedTocDiagnostics;
limits: OnePkgLimits;
budget: TocBuildBudget;
activeTocs: Set<string>; activeTocs: Set<string>;
usedTocs: Set<string>; usedTocs: Set<string>;
depth: number; depth: number;
} }
function buildTocNodes( async function buildTocNodes(
context: BuildTocNodesContext context: BuildTocNodesContext
): OneNoteNotebookNodeDto[] { ): Promise<OneNoteNotebookNodeDto[]> {
if (context.depth > DEFAULT_ONEPKG_LIMITS.maxPathDepth) { if (context.depth > context.limits.maxPathDepth) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-tree-depth-limit', 'onetoc2-tree-depth-limit',
'Notebook section-group nesting exceeds the package path-depth limit.', 'Notebook section-group nesting exceeds the package path-depth limit.',
@@ -170,7 +237,7 @@ function buildTocNodes(
} }
const tocKey = pathKey(context.parsed.extracted.normalizedPath); const tocKey = pathKey(context.parsed.extracted.normalizedPath);
if (context.activeTocs.has(tocKey)) { if (context.activeTocs.has(tocKey)) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-tree-cycle', 'onetoc2-tree-cycle',
'A cycle between notebook table-of-contents files was ignored.', 'A cycle between notebook table-of-contents files was ignored.',
@@ -180,7 +247,7 @@ function buildTocNodes(
return []; return [];
} }
if (context.usedTocs.has(tocKey)) { if (context.usedTocs.has(tocKey)) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-tree-duplicate', 'onetoc2-tree-duplicate',
'A repeated notebook section-group reference was ignored.', 'A repeated notebook section-group reference was ignored.',
@@ -195,16 +262,19 @@ function buildTocNodes(
try { try {
const result: OneNoteNotebookNodeDto[] = []; const result: OneNoteNotebookNodeDto[] = [];
for (const entry of context.parsed.toc.entries) { for (const entry of context.parsed.toc.entries) {
const checkpoint = context.budget.charge();
if (checkpoint) await checkpoint;
if (isRecycleBinName(entry.filename)) continue; if (isRecycleBinName(entry.filename)) continue;
try { try {
const childPath = resolveTocChildPath( const childPath = resolveTocChildPath(
context.parsed.directory, context.parsed.directory,
entry.filename entry.filename,
context.limits
); );
const section = resolveSection(context, childPath, entry); const section = resolveSection(context, childPath, entry);
if (section) { if (section) {
if (context.usedSectionIds.has(section.id)) { if (context.usedSectionIds.has(section.id)) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-section-duplicate', 'onetoc2-section-duplicate',
`Repeated TOC reference to ${entry.filename} was ignored.`, `Repeated TOC reference to ${entry.filename} was ignored.`,
@@ -229,7 +299,7 @@ function buildTocNodes(
if (childToc) { if (childToc) {
const childTocKey = pathKey(childToc.extracted.normalizedPath); const childTocKey = pathKey(childToc.extracted.normalizedPath);
if (context.activeTocs.has(childTocKey)) { if (context.activeTocs.has(childTocKey)) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-tree-cycle', 'onetoc2-tree-cycle',
'A cycle between notebook table-of-contents files was ignored.', 'A cycle between notebook table-of-contents files was ignored.',
@@ -239,7 +309,7 @@ function buildTocNodes(
continue; continue;
} }
if (context.usedTocs.has(childTocKey)) { if (context.usedTocs.has(childTocKey)) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-tree-duplicate', 'onetoc2-tree-duplicate',
'A repeated notebook section-group reference was ignored.', 'A repeated notebook section-group reference was ignored.',
@@ -254,7 +324,7 @@ function buildTocNodes(
displayName: leafName(childPath), displayName: leafName(childPath),
orderingId: entry.orderingId, orderingId: entry.orderingId,
color: entry.color, color: entry.color,
children: buildTocNodes({ children: await buildTocNodes({
...context, ...context,
parsed: childToc, parsed: childToc,
depth: context.depth + 1, depth: context.depth + 1,
@@ -263,7 +333,7 @@ function buildTocNodes(
continue; continue;
} }
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-entry-unresolved', 'onetoc2-entry-unresolved',
`TOC entry ${entry.filename} has no matching section or section group in the package.`, `TOC entry ${entry.filename} has no matching section or section group in the package.`,
@@ -271,7 +341,8 @@ function buildTocNodes(
) )
); );
} catch (error) { } catch (error) {
context.diagnostics.push( if (isFatalTocBuildError(error)) throw error;
context.diagnostics.add(
tocChildFailureDiagnostic( tocChildFailureDiagnostic(
error, error,
context.parsed.extracted.normalizedPath, context.parsed.extracted.normalizedPath,
@@ -296,7 +367,7 @@ function resolveSection(
if (exact) { if (exact) {
const identity = context.sectionIdentityByPath.get(expectedKey); const identity = context.sectionIdentityByPath.get(expectedKey);
if (identity && identity !== tocEntry.fileIdentity) { if (identity && identity !== tocEntry.fileIdentity) {
context.diagnostics.push( context.diagnostics.add(
warning( warning(
'onetoc2-section-identity-mismatch', 'onetoc2-section-identity-mismatch',
`TOC identity for ${tocEntry.filename} does not match the section header; the package filename takes precedence.`, `TOC identity for ${tocEntry.filename} does not match the section header; the package filename takes precedence.`,
@@ -307,47 +378,54 @@ function resolveSection(
return exact; return exact;
} }
const directory = parentPath(childPath); const match = context.sectionByDirectoryAndIdentity
const identityMatches = context.sections.filter( .get(pathKey(parentPath(childPath)))
(section) => ?.get(tocEntry.fileIdentity);
pathKey(parentPath(section.path)) === pathKey(directory) && if (match && !match.ambiguous) {
context.sectionIdentityByPath.get(pathKey(section.path)) === context.diagnostics.add(
tocEntry.fileIdentity
);
if (identityMatches.length === 1) {
const match = identityMatches[0]!;
context.diagnostics.push(
warning( warning(
'onetoc2-section-filename-mismatch', 'onetoc2-section-filename-mismatch',
`TOC filename ${tocEntry.filename} was resolved to package section ${leafName(match.path)} by FileIdentityGuid.`, `TOC filename ${tocEntry.filename} was resolved to package section ${leafName(match.section.path)} by FileIdentityGuid.`,
{ structure: context.parsed.extracted.normalizedPath } { structure: context.parsed.extracted.normalizedPath }
) )
); );
return match; return match.section;
} }
return undefined; return undefined;
} }
function selectRootToc( async function selectRootToc(
parsedTocs: ParsedTocEntry[], parsedTocs: ParsedTocEntry[],
diagnostics: ParserDiagnostic[] diagnostics: CappedTocDiagnostics,
): ParsedTocEntry | undefined { budget: TocBuildBudget
): Promise<ParsedTocEntry | undefined> {
if (parsedTocs.length === 0) return undefined; if (parsedTocs.length === 0) return undefined;
const sorted = [...parsedTocs].sort((left, right) => { let selected: ParsedTocEntry | undefined;
const depth = pathDepth(left.directory) - pathDepth(right.directory); let minimumDepth = Number.POSITIVE_INFINITY;
return ( let candidateCount = 0;
depth || for (const candidate of parsedTocs) {
left.extracted.normalizedPath.localeCompare( const checkpoint = budget.charge();
right.extracted.normalizedPath if (checkpoint) await checkpoint;
) const depth = pathDepth(candidate.directory);
); if (depth < minimumDepth) {
}); selected = candidate;
const minimumDepth = pathDepth(sorted[0]!.directory); minimumDepth = depth;
const candidates = sorted.filter( candidateCount = 1;
(candidate) => pathDepth(candidate.directory) === minimumDepth continue;
); }
if (candidates.length > 1) { if (depth !== minimumDepth) continue;
diagnostics.push( candidateCount += 1;
if (
selected === undefined ||
candidate.extracted.normalizedPath.localeCompare(
selected.extracted.normalizedPath
) < 0
) {
selected = candidate;
}
}
if (candidateCount > 1) {
diagnostics.add(
warning( warning(
'onetoc2-root-ambiguous', 'onetoc2-root-ambiguous',
'More than one top-level table of contents was found; the first safe package path is used.', 'More than one top-level table of contents was found; the first safe package path is used.',
@@ -355,12 +433,16 @@ function selectRootToc(
) )
); );
} }
return candidates[0]; return selected;
} }
export function resolveTocChildPath( export function resolveTocChildPath(
directory: string, directory: string,
filename: string filename: string,
limits: Pick<
OnePkgLimits,
'maxPathCharacters' | 'maxPathDepth'
> = DEFAULT_ONEPKG_LIMITS
): string { ): string {
if ( if (
filename.length === 0 || filename.length === 0 ||
@@ -381,8 +463,8 @@ export function resolveTocChildPath(
const normalized = filename.normalize('NFC'); const normalized = filename.normalize('NFC');
const path = directory ? `${directory}/${normalized}` : normalized; const path = directory ? `${directory}/${normalized}` : normalized;
if ( if (
path.length > DEFAULT_ONEPKG_LIMITS.maxPathCharacters || path.length > limits.maxPathCharacters ||
pathDepth(path) > DEFAULT_ONEPKG_LIMITS.maxPathDepth pathDepth(path) > limits.maxPathDepth
) { ) {
fail( fail(
'onetoc2-child-path-limit', 'onetoc2-child-path-limit',
@@ -450,6 +532,75 @@ function tocChildFailureDiagnostic(
}; };
} }
function isFatalTocBuildError(error: unknown): boolean {
return (
error instanceof OnePkgError &&
(error.diagnostic.code === 'onepkg-cancelled' ||
error.diagnostic.code === 'onetoc2-operation-limit')
);
}
const TOC_CANCELLATION_INTERVAL = 256;
class TocBuildBudget {
private operations = 0;
private nextCancellationCheck = TOC_CANCELLATION_INTERVAL;
constructor(
private readonly limits: OnePkgLimits,
private readonly cancellation: OnePkgCancellation | undefined
) {
checkCancellation(cancellation);
}
charge(): Promise<void> | undefined {
this.chargeWithoutYield();
if (this.operations < this.nextCancellationCheck) return undefined;
this.nextCancellationCheck = this.operations + TOC_CANCELLATION_INTERVAL;
return yieldForCancellation(this.cancellation);
}
chargeWithoutYield(): void {
if (this.operations >= this.limits.maxTocOperations) {
fail(
'onetoc2-operation-limit',
`Building the notebook hierarchy exceeds ${this.limits.maxTocOperations} bounded operations.`,
{ structure: 'ONEPKG table of contents' }
);
}
this.operations += 1;
}
async finish(): Promise<void> {
await yieldForCancellation(this.cancellation);
}
}
class CappedTocDiagnostics {
private readonly diagnostics: ParserDiagnostic[] = [];
private truncated = false;
constructor(private readonly limit: number) {}
add(diagnostic: ParserDiagnostic): void {
if (this.truncated) return;
if (this.diagnostics.length < this.limit) {
this.diagnostics.push(diagnostic);
return;
}
this.truncated = true;
this.diagnostics[this.limit - 1] = warning(
'onetoc2-diagnostic-limit',
`Additional notebook hierarchy diagnostics were omitted after reaching the limit of ${this.limit}.`,
{ structure: 'ONEPKG table of contents' }
);
}
values(): ParserDiagnostic[] {
return [...this.diagnostics];
}
}
function parentPath(path: string): string { function parentPath(path: string): string {
const slash = path.lastIndexOf('/'); const slash = path.lastIndexOf('/');
return slash < 0 ? '' : path.slice(0, slash); return slash < 0 ? '' : path.slice(0, slash);

View File

@@ -90,6 +90,10 @@ export interface OnePkgLimits {
maxNameBytes: number; maxNameBytes: number;
maxPathCharacters: number; maxPathCharacters: number;
maxPathDepth: number; maxPathDepth: number;
/** Maximum indexed TOC hierarchy construction and resolution steps. */
maxTocOperations: number;
/** Maximum package-level diagnostics emitted while interpreting TOCs. */
maxTocDiagnostics: number;
maxFileBytes: number; maxFileBytes: number;
maxFolderUncompressedBytes: number; maxFolderUncompressedBytes: number;
maxTotalUncompressedBytes: number; maxTotalUncompressedBytes: number;
@@ -111,6 +115,8 @@ export const DEFAULT_ONEPKG_LIMITS: Readonly<OnePkgLimits> = {
maxNameBytes: 255, maxNameBytes: 255,
maxPathCharacters: 4_096, maxPathCharacters: 4_096,
maxPathDepth: 64, maxPathDepth: 64,
maxTocOperations: 1_000_000,
maxTocDiagnostics: 10_000,
maxFileBytes: 256 * 1024 * 1024, maxFileBytes: 256 * 1024 * 1024,
maxFolderUncompressedBytes: 512 * 1024 * 1024, maxFolderUncompressedBytes: 512 * 1024 * 1024,
maxTotalUncompressedBytes: 512 * 1024 * 1024, maxTotalUncompressedBytes: 512 * 1024 * 1024,