perf: bound OneNote TOC hierarchy building
This commit is contained in:
@@ -194,9 +194,10 @@ async function buildOpenedPackage(
|
||||
});
|
||||
}
|
||||
|
||||
const treeResult = buildOneNotePackageTree(
|
||||
const treeResult = await buildOneNotePackageTree(
|
||||
extraction.extractedEntries,
|
||||
packagedSections
|
||||
packagedSections,
|
||||
options
|
||||
);
|
||||
diagnostics.push(...treeResult.diagnostics);
|
||||
|
||||
|
||||
@@ -1,6 +1,29 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { resolveTocChildPath } from './toc-hierarchy.js';
|
||||
import type { OneNotePackagedSectionDto } from '../model/dto.js';
|
||||
import type {
|
||||
OneNoteTableOfContents,
|
||||
OneNoteTableOfContentsEntry,
|
||||
} from '../parser/parse-toc.js';
|
||||
import {
|
||||
DESKTOP_REVISION_STORE_FORMAT,
|
||||
ONE_SECTION_FILE_TYPE,
|
||||
} from '../parser/detect-format.js';
|
||||
import type { OnePkgError } from './errors.js';
|
||||
import {
|
||||
buildOneNotePackageTree,
|
||||
resolveTocChildPath,
|
||||
} from './toc-hierarchy.js';
|
||||
import type { ExtractedCabFile } from './types.js';
|
||||
|
||||
const parseTocMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock('../parser/parse-toc.js', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('../parser/parse-toc.js')>()),
|
||||
parseOneNoteTableOfContents: parseTocMock,
|
||||
}));
|
||||
|
||||
beforeEach(() => parseTocMock.mockReset());
|
||||
|
||||
describe('ONEPKG TOC child paths', () => {
|
||||
it('joins a single NFC-normalized child name within its section group', () => {
|
||||
@@ -18,3 +41,244 @@ describe('ONEPKG TOC child paths', () => {
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe('ONEPKG TOC hierarchy budgets and indexes', () => {
|
||||
it('resolves identity fallbacks in authored order with near-linear section lookups', async () => {
|
||||
const count = 400;
|
||||
const pathReads = { count: 0 };
|
||||
const entries: OneNoteTableOfContentsEntry[] = [];
|
||||
const extracted: ExtractedCabFile[] = [tocFile()];
|
||||
const sections: OneNotePackagedSectionDto[] = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const identity = indexedGuid(index + 1);
|
||||
const path = `Section-${index.toString().padStart(4, '0')}.one`;
|
||||
extracted.push(sectionFile(path, identity, index + 1));
|
||||
sections.push(sectionWithObservedPath(path, pathReads));
|
||||
entries.push({
|
||||
filename: `Alias-${index.toString().padStart(4, '0')}.one`,
|
||||
fileIdentity: identity,
|
||||
orderingId: index,
|
||||
});
|
||||
}
|
||||
parseTocMock.mockReturnValue(toc(entries));
|
||||
let yields = 0;
|
||||
|
||||
const result = await buildOneNotePackageTree(extracted, sections, {
|
||||
limits: { maxTocOperations: count * 8 + 100 },
|
||||
cancellation: {
|
||||
yield: () => {
|
||||
yields += 1;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.tree.nodes).toHaveLength(count);
|
||||
expect(
|
||||
result.tree.nodes.map((node) =>
|
||||
node.kind === 'section' ? node.sectionId : `group:${node.id}`
|
||||
)
|
||||
).toEqual(sections.map(({ id }) => id));
|
||||
expect(pathReads.count).toBeLessThanOrEqual(count * 5);
|
||||
expect(yields).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('keeps ambiguous identities unresolved and fallback sections sorted', async () => {
|
||||
const identity = indexedGuid(42);
|
||||
parseTocMock.mockReturnValue(
|
||||
toc([{ filename: 'Alias.one', fileIdentity: identity, orderingId: 1 }])
|
||||
);
|
||||
const sections = [section('Z.one'), section('A.one')];
|
||||
|
||||
const result = await buildOneNotePackageTree(
|
||||
[
|
||||
tocFile(),
|
||||
sectionFile('Z.one', identity, 1),
|
||||
sectionFile('A.one', identity, 2),
|
||||
],
|
||||
sections
|
||||
);
|
||||
|
||||
expect(
|
||||
result.tree.nodes.map((node) =>
|
||||
node.kind === 'section' ? node.sectionId : `group:${node.id}`
|
||||
)
|
||||
).toEqual(['A.one', 'Z.one']);
|
||||
expect(result.diagnostics.map(({ code }) => code)).toEqual(
|
||||
expect.arrayContaining([
|
||||
'onetoc2-entry-unresolved',
|
||||
'onetoc2-unlisted-sections',
|
||||
])
|
||||
);
|
||||
expect(result.diagnostics).not.toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
code: 'onetoc2-section-filename-mismatch',
|
||||
}),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it('caps adversarial unresolved-entry diagnostics with an explicit marker', async () => {
|
||||
parseTocMock.mockReturnValue(
|
||||
toc(
|
||||
Array.from({ length: 50 }, (_, index) => ({
|
||||
filename: `Missing-${index}.one`,
|
||||
fileIdentity: indexedGuid(index + 1),
|
||||
orderingId: index,
|
||||
}))
|
||||
)
|
||||
);
|
||||
|
||||
const result = await buildOneNotePackageTree([tocFile()], [], {
|
||||
limits: { maxTocDiagnostics: 5, maxTocOperations: 1_000 },
|
||||
});
|
||||
|
||||
expect(result.diagnostics).toHaveLength(5);
|
||||
expect(result.diagnostics.slice(0, 4).map(({ code }) => code)).toEqual([
|
||||
'onetoc2-entry-unresolved',
|
||||
'onetoc2-entry-unresolved',
|
||||
'onetoc2-entry-unresolved',
|
||||
'onetoc2-entry-unresolved',
|
||||
]);
|
||||
expect(result.diagnostics[4]?.code).toBe('onetoc2-diagnostic-limit');
|
||||
});
|
||||
|
||||
it('fails closed at the hierarchy operation limit', async () => {
|
||||
parseTocMock.mockReturnValue(
|
||||
toc(
|
||||
Array.from({ length: 20 }, (_, index) => ({
|
||||
filename: `Missing-${index}.one`,
|
||||
fileIdentity: indexedGuid(index + 1),
|
||||
orderingId: index,
|
||||
}))
|
||||
)
|
||||
);
|
||||
|
||||
await expect(
|
||||
buildOneNotePackageTree([tocFile()], [], {
|
||||
limits: { maxTocOperations: 10 },
|
||||
})
|
||||
).rejects.toEqual(
|
||||
expect.objectContaining<Partial<OnePkgError>>({
|
||||
diagnostic: expect.objectContaining({
|
||||
code: 'onetoc2-operation-limit',
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('yields and observes cancellation during a large hierarchy build', async () => {
|
||||
parseTocMock.mockReturnValue(
|
||||
toc(
|
||||
Array.from({ length: 1_000 }, (_, index) => ({
|
||||
filename: `Missing-${index}.one`,
|
||||
fileIdentity: indexedGuid(index + 1),
|
||||
orderingId: index,
|
||||
}))
|
||||
)
|
||||
);
|
||||
const controller = new AbortController();
|
||||
let yields = 0;
|
||||
|
||||
await expect(
|
||||
buildOneNotePackageTree([tocFile()], [], {
|
||||
cancellation: {
|
||||
signal: controller.signal,
|
||||
yield: () => {
|
||||
yields += 1;
|
||||
controller.abort();
|
||||
},
|
||||
},
|
||||
})
|
||||
).rejects.toEqual(
|
||||
expect.objectContaining<Partial<OnePkgError>>({
|
||||
diagnostic: expect.objectContaining({ code: 'onepkg-cancelled' }),
|
||||
})
|
||||
);
|
||||
expect(yields).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
function toc(entries: OneNoteTableOfContentsEntry[]): OneNoteTableOfContents {
|
||||
return {
|
||||
sourceName: 'Open Notebook.onetoc2',
|
||||
fileIdentity: indexedGuid(0),
|
||||
entries,
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
function tocFile(): ExtractedCabFile {
|
||||
return extractedFile('Open Notebook.onetoc2', Uint8Array.of(1), 0);
|
||||
}
|
||||
|
||||
function sectionFile(
|
||||
path: string,
|
||||
identity: string,
|
||||
index: number
|
||||
): ExtractedCabFile {
|
||||
const bytes = new Uint8Array(64);
|
||||
writeGuid(bytes, 0, ONE_SECTION_FILE_TYPE);
|
||||
writeGuid(bytes, 16, identity);
|
||||
writeGuid(bytes, 48, DESKTOP_REVISION_STORE_FORMAT);
|
||||
return extractedFile(path, bytes, index);
|
||||
}
|
||||
|
||||
function extractedFile(
|
||||
path: string,
|
||||
bytes: Uint8Array,
|
||||
index: number
|
||||
): ExtractedCabFile {
|
||||
return {
|
||||
index,
|
||||
path,
|
||||
normalizedPath: path,
|
||||
folderIndex: 0,
|
||||
uncompressedOffset: 0,
|
||||
uncompressedSize: bytes.byteLength,
|
||||
attributes: 0,
|
||||
compression: { kind: 'none', raw: 0, label: 'None' },
|
||||
bytes,
|
||||
};
|
||||
}
|
||||
|
||||
function section(path: string): OneNotePackagedSectionDto {
|
||||
return {
|
||||
id: path,
|
||||
path,
|
||||
displayName: path.slice(0, -4),
|
||||
parseStatus: 'not-requested',
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
function sectionWithObservedPath(
|
||||
path: string,
|
||||
reads: { count: number }
|
||||
): OneNotePackagedSectionDto {
|
||||
const result = section(path);
|
||||
Object.defineProperty(result, 'path', {
|
||||
enumerable: true,
|
||||
get: () => {
|
||||
reads.count += 1;
|
||||
return path;
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
function indexedGuid(index: number): string {
|
||||
const plain = index.toString(16).toUpperCase().padStart(32, '0');
|
||||
return `{${plain.slice(0, 8)}-${plain.slice(8, 12)}-${plain.slice(12, 16)}-${plain.slice(16, 20)}-${plain.slice(20)}}`;
|
||||
}
|
||||
|
||||
function writeGuid(target: Uint8Array, offset: number, guid: string): void {
|
||||
const plain = guid.replace(/[{}-]/gu, '');
|
||||
const canonical = Array.from({ length: 16 }, (_, index) =>
|
||||
Number.parseInt(plain.slice(index * 2, index * 2 + 2), 16)
|
||||
);
|
||||
const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15];
|
||||
for (let index = 0; index < order.length; index += 1) {
|
||||
target[offset + order[index]!] = canonical[index]!;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,9 +16,16 @@ import {
|
||||
type OneNoteTableOfContents,
|
||||
type OneNoteTableOfContentsEntry,
|
||||
} from '../parser/parse-toc.js';
|
||||
import { checkCancellation, yieldForCancellation } from './cancellation.js';
|
||||
import { fail, OnePkgError, warning } from './errors.js';
|
||||
import type { ExtractedCabFile } from './types.js';
|
||||
import { DEFAULT_ONEPKG_LIMITS } from './types.js';
|
||||
import {
|
||||
DEFAULT_ONEPKG_LIMITS,
|
||||
resolveLimits,
|
||||
type CabOpenOptions,
|
||||
type ExtractedCabFile,
|
||||
type OnePkgCancellation,
|
||||
type OnePkgLimits,
|
||||
} from './types.js';
|
||||
|
||||
interface ParsedTocEntry {
|
||||
extracted: ExtractedCabFile;
|
||||
@@ -35,18 +42,24 @@ export interface OneNotePackageTreeResult {
|
||||
* Interpret package TOCs and join them to extracted sections without ever
|
||||
* materializing an archive path on the host filesystem.
|
||||
*/
|
||||
export function buildOneNotePackageTree(
|
||||
export async function buildOneNotePackageTree(
|
||||
extractedEntries: ExtractedCabFile[],
|
||||
sections: OneNotePackagedSectionDto[]
|
||||
): OneNotePackageTreeResult {
|
||||
const diagnostics: ParserDiagnostic[] = [];
|
||||
sections: OneNotePackagedSectionDto[],
|
||||
options: CabOpenOptions = {}
|
||||
): Promise<OneNotePackageTreeResult> {
|
||||
const limits = resolveLimits(options.limits);
|
||||
const budget = new TocBuildBudget(limits, options.cancellation);
|
||||
const diagnostics = new CappedTocDiagnostics(limits.maxTocDiagnostics);
|
||||
const parsedTocs: ParsedTocEntry[] = [];
|
||||
|
||||
for (const extracted of extractedEntries) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
if (!extracted.normalizedPath.toLowerCase().endsWith('.onetoc2')) continue;
|
||||
try {
|
||||
const toc = parseOneNoteTableOfContents(extracted.bytes, {
|
||||
sourceName: extracted.normalizedPath,
|
||||
limits: { maxDiagnostics: limits.maxTocDiagnostics },
|
||||
});
|
||||
parsedTocs.push({
|
||||
extracted,
|
||||
@@ -54,8 +67,9 @@ export function buildOneNotePackageTree(
|
||||
directory: parentPath(extracted.normalizedPath),
|
||||
});
|
||||
for (const diagnostic of toc.diagnostics) {
|
||||
if (diagnostics.length >= DEFAULT_ONEPKG_LIMITS.maxFiles) break;
|
||||
diagnostics.push({
|
||||
const diagnosticCheckpoint = budget.charge();
|
||||
if (diagnosticCheckpoint) await diagnosticCheckpoint;
|
||||
diagnostics.add({
|
||||
...diagnostic,
|
||||
structure: diagnostic.structure
|
||||
? `${extracted.normalizedPath} · ${diagnostic.structure}`
|
||||
@@ -63,29 +77,63 @@ export function buildOneNotePackageTree(
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
diagnostics.push(tocFailureDiagnostic(error, extracted.normalizedPath));
|
||||
if (isFatalTocBuildError(error)) throw error;
|
||||
diagnostics.add(tocFailureDiagnostic(error, extracted.normalizedPath));
|
||||
}
|
||||
}
|
||||
|
||||
const root = selectRootToc(parsedTocs, diagnostics);
|
||||
const sectionByPath = new Map(
|
||||
sections.map((section) => [pathKey(section.path), section])
|
||||
);
|
||||
const extractedSectionByPath = new Map(
|
||||
extractedEntries
|
||||
.filter((entry) => entry.normalizedPath.toLowerCase().endsWith('.one'))
|
||||
.map((entry) => [pathKey(entry.normalizedPath), entry])
|
||||
);
|
||||
const root = await selectRootToc(parsedTocs, diagnostics, budget);
|
||||
const sectionByPath = new Map<string, OneNotePackagedSectionDto>();
|
||||
for (const section of sections) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
sectionByPath.set(pathKey(section.path), section);
|
||||
}
|
||||
const extractedSectionByPath = new Map<string, ExtractedCabFile>();
|
||||
for (const entry of extractedEntries) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
if (!entry.normalizedPath.toLowerCase().endsWith('.one')) continue;
|
||||
extractedSectionByPath.set(pathKey(entry.normalizedPath), entry);
|
||||
}
|
||||
const sectionIdentityByPath = new Map<string, string>();
|
||||
for (const [key, entry] of extractedSectionByPath) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
const identity = readSectionIdentity(entry.bytes);
|
||||
if (identity) sectionIdentityByPath.set(key, identity);
|
||||
}
|
||||
const sectionByDirectoryAndIdentity = new Map<
|
||||
string,
|
||||
Map<string, IdentitySectionMatch>
|
||||
>();
|
||||
for (const section of sections) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
const sectionPathKey = pathKey(section.path);
|
||||
const identity = sectionIdentityByPath.get(sectionPathKey);
|
||||
if (!identity) continue;
|
||||
const directoryKey = pathKey(parentPath(section.path));
|
||||
let byIdentity = sectionByDirectoryAndIdentity.get(directoryKey);
|
||||
if (!byIdentity) {
|
||||
byIdentity = new Map();
|
||||
sectionByDirectoryAndIdentity.set(directoryKey, byIdentity);
|
||||
}
|
||||
const existing = byIdentity.get(identity);
|
||||
byIdentity.set(
|
||||
identity,
|
||||
existing
|
||||
? { section: existing.section, ambiguous: true }
|
||||
: { section, ambiguous: false }
|
||||
);
|
||||
}
|
||||
const tocByDirectory = new Map<string, ParsedTocEntry>();
|
||||
for (const parsed of parsedTocs) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
const key = pathKey(parsed.directory);
|
||||
if (tocByDirectory.has(key)) {
|
||||
diagnostics.push(
|
||||
diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-directory-ambiguous',
|
||||
`More than one readable .onetoc2 exists in ${parsed.directory || 'the package root'}; the first one is used.`,
|
||||
@@ -100,28 +148,36 @@ export function buildOneNotePackageTree(
|
||||
const usedSectionIds = new Set<string>();
|
||||
let nodes: OneNoteNotebookNodeDto[] = [];
|
||||
if (root) {
|
||||
nodes = buildTocNodes({
|
||||
nodes = await buildTocNodes({
|
||||
parsed: root,
|
||||
tocByDirectory,
|
||||
sectionByPath,
|
||||
sectionIdentityByPath,
|
||||
sections,
|
||||
sectionByDirectoryAndIdentity,
|
||||
usedSectionIds,
|
||||
diagnostics,
|
||||
limits,
|
||||
budget,
|
||||
activeTocs: new Set(),
|
||||
usedTocs: new Set(),
|
||||
depth: 0,
|
||||
});
|
||||
}
|
||||
|
||||
const unlisted = sections
|
||||
.filter(
|
||||
(section) =>
|
||||
!usedSectionIds.has(section.id) && !isRecycleBinPath(section.path)
|
||||
)
|
||||
.sort((left, right) => left.path.localeCompare(right.path));
|
||||
const unlisted: OneNotePackagedSectionDto[] = [];
|
||||
for (const section of sections) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
if (!usedSectionIds.has(section.id) && !isRecycleBinPath(section.path)) {
|
||||
unlisted.push(section);
|
||||
}
|
||||
}
|
||||
unlisted.sort((left, right) => {
|
||||
budget.chargeWithoutYield();
|
||||
return left.path.localeCompare(right.path);
|
||||
});
|
||||
if (unlisted.length > 0 && root) {
|
||||
diagnostics.push(
|
||||
diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-unlisted-sections',
|
||||
`${unlisted.length} package section${unlisted.length === 1 ? '' : 's'} were not reachable from the parsed TOC and were appended to navigation.`,
|
||||
@@ -130,6 +186,7 @@ export function buildOneNotePackageTree(
|
||||
);
|
||||
}
|
||||
nodes.push(...unlisted.map(fallbackSectionNode));
|
||||
await budget.finish();
|
||||
|
||||
return {
|
||||
tree: {
|
||||
@@ -138,28 +195,38 @@ export function buildOneNotePackageTree(
|
||||
color: root?.toc.color,
|
||||
nodes,
|
||||
},
|
||||
diagnostics,
|
||||
diagnostics: diagnostics.values(),
|
||||
};
|
||||
}
|
||||
|
||||
interface IdentitySectionMatch {
|
||||
section: OneNotePackagedSectionDto;
|
||||
ambiguous: boolean;
|
||||
}
|
||||
|
||||
interface BuildTocNodesContext {
|
||||
parsed: ParsedTocEntry;
|
||||
tocByDirectory: ReadonlyMap<string, ParsedTocEntry>;
|
||||
sectionByPath: ReadonlyMap<string, OneNotePackagedSectionDto>;
|
||||
sectionIdentityByPath: ReadonlyMap<string, string>;
|
||||
sections: OneNotePackagedSectionDto[];
|
||||
sectionByDirectoryAndIdentity: ReadonlyMap<
|
||||
string,
|
||||
ReadonlyMap<string, IdentitySectionMatch>
|
||||
>;
|
||||
usedSectionIds: Set<string>;
|
||||
diagnostics: ParserDiagnostic[];
|
||||
diagnostics: CappedTocDiagnostics;
|
||||
limits: OnePkgLimits;
|
||||
budget: TocBuildBudget;
|
||||
activeTocs: Set<string>;
|
||||
usedTocs: Set<string>;
|
||||
depth: number;
|
||||
}
|
||||
|
||||
function buildTocNodes(
|
||||
async function buildTocNodes(
|
||||
context: BuildTocNodesContext
|
||||
): OneNoteNotebookNodeDto[] {
|
||||
if (context.depth > DEFAULT_ONEPKG_LIMITS.maxPathDepth) {
|
||||
context.diagnostics.push(
|
||||
): Promise<OneNoteNotebookNodeDto[]> {
|
||||
if (context.depth > context.limits.maxPathDepth) {
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-tree-depth-limit',
|
||||
'Notebook section-group nesting exceeds the package path-depth limit.',
|
||||
@@ -170,7 +237,7 @@ function buildTocNodes(
|
||||
}
|
||||
const tocKey = pathKey(context.parsed.extracted.normalizedPath);
|
||||
if (context.activeTocs.has(tocKey)) {
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-tree-cycle',
|
||||
'A cycle between notebook table-of-contents files was ignored.',
|
||||
@@ -180,7 +247,7 @@ function buildTocNodes(
|
||||
return [];
|
||||
}
|
||||
if (context.usedTocs.has(tocKey)) {
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-tree-duplicate',
|
||||
'A repeated notebook section-group reference was ignored.',
|
||||
@@ -195,16 +262,19 @@ function buildTocNodes(
|
||||
try {
|
||||
const result: OneNoteNotebookNodeDto[] = [];
|
||||
for (const entry of context.parsed.toc.entries) {
|
||||
const checkpoint = context.budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
if (isRecycleBinName(entry.filename)) continue;
|
||||
try {
|
||||
const childPath = resolveTocChildPath(
|
||||
context.parsed.directory,
|
||||
entry.filename
|
||||
entry.filename,
|
||||
context.limits
|
||||
);
|
||||
const section = resolveSection(context, childPath, entry);
|
||||
if (section) {
|
||||
if (context.usedSectionIds.has(section.id)) {
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-section-duplicate',
|
||||
`Repeated TOC reference to ${entry.filename} was ignored.`,
|
||||
@@ -229,7 +299,7 @@ function buildTocNodes(
|
||||
if (childToc) {
|
||||
const childTocKey = pathKey(childToc.extracted.normalizedPath);
|
||||
if (context.activeTocs.has(childTocKey)) {
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-tree-cycle',
|
||||
'A cycle between notebook table-of-contents files was ignored.',
|
||||
@@ -239,7 +309,7 @@ function buildTocNodes(
|
||||
continue;
|
||||
}
|
||||
if (context.usedTocs.has(childTocKey)) {
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-tree-duplicate',
|
||||
'A repeated notebook section-group reference was ignored.',
|
||||
@@ -254,7 +324,7 @@ function buildTocNodes(
|
||||
displayName: leafName(childPath),
|
||||
orderingId: entry.orderingId,
|
||||
color: entry.color,
|
||||
children: buildTocNodes({
|
||||
children: await buildTocNodes({
|
||||
...context,
|
||||
parsed: childToc,
|
||||
depth: context.depth + 1,
|
||||
@@ -263,7 +333,7 @@ function buildTocNodes(
|
||||
continue;
|
||||
}
|
||||
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-entry-unresolved',
|
||||
`TOC entry ${entry.filename} has no matching section or section group in the package.`,
|
||||
@@ -271,7 +341,8 @@ function buildTocNodes(
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
context.diagnostics.push(
|
||||
if (isFatalTocBuildError(error)) throw error;
|
||||
context.diagnostics.add(
|
||||
tocChildFailureDiagnostic(
|
||||
error,
|
||||
context.parsed.extracted.normalizedPath,
|
||||
@@ -296,7 +367,7 @@ function resolveSection(
|
||||
if (exact) {
|
||||
const identity = context.sectionIdentityByPath.get(expectedKey);
|
||||
if (identity && identity !== tocEntry.fileIdentity) {
|
||||
context.diagnostics.push(
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-section-identity-mismatch',
|
||||
`TOC identity for ${tocEntry.filename} does not match the section header; the package filename takes precedence.`,
|
||||
@@ -307,47 +378,54 @@ function resolveSection(
|
||||
return exact;
|
||||
}
|
||||
|
||||
const directory = parentPath(childPath);
|
||||
const identityMatches = context.sections.filter(
|
||||
(section) =>
|
||||
pathKey(parentPath(section.path)) === pathKey(directory) &&
|
||||
context.sectionIdentityByPath.get(pathKey(section.path)) ===
|
||||
tocEntry.fileIdentity
|
||||
);
|
||||
if (identityMatches.length === 1) {
|
||||
const match = identityMatches[0]!;
|
||||
context.diagnostics.push(
|
||||
const match = context.sectionByDirectoryAndIdentity
|
||||
.get(pathKey(parentPath(childPath)))
|
||||
?.get(tocEntry.fileIdentity);
|
||||
if (match && !match.ambiguous) {
|
||||
context.diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-section-filename-mismatch',
|
||||
`TOC filename ${tocEntry.filename} was resolved to package section ${leafName(match.path)} by FileIdentityGuid.`,
|
||||
`TOC filename ${tocEntry.filename} was resolved to package section ${leafName(match.section.path)} by FileIdentityGuid.`,
|
||||
{ structure: context.parsed.extracted.normalizedPath }
|
||||
)
|
||||
);
|
||||
return match;
|
||||
return match.section;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function selectRootToc(
|
||||
async function selectRootToc(
|
||||
parsedTocs: ParsedTocEntry[],
|
||||
diagnostics: ParserDiagnostic[]
|
||||
): ParsedTocEntry | undefined {
|
||||
diagnostics: CappedTocDiagnostics,
|
||||
budget: TocBuildBudget
|
||||
): Promise<ParsedTocEntry | undefined> {
|
||||
if (parsedTocs.length === 0) return undefined;
|
||||
const sorted = [...parsedTocs].sort((left, right) => {
|
||||
const depth = pathDepth(left.directory) - pathDepth(right.directory);
|
||||
return (
|
||||
depth ||
|
||||
left.extracted.normalizedPath.localeCompare(
|
||||
right.extracted.normalizedPath
|
||||
)
|
||||
);
|
||||
});
|
||||
const minimumDepth = pathDepth(sorted[0]!.directory);
|
||||
const candidates = sorted.filter(
|
||||
(candidate) => pathDepth(candidate.directory) === minimumDepth
|
||||
);
|
||||
if (candidates.length > 1) {
|
||||
diagnostics.push(
|
||||
let selected: ParsedTocEntry | undefined;
|
||||
let minimumDepth = Number.POSITIVE_INFINITY;
|
||||
let candidateCount = 0;
|
||||
for (const candidate of parsedTocs) {
|
||||
const checkpoint = budget.charge();
|
||||
if (checkpoint) await checkpoint;
|
||||
const depth = pathDepth(candidate.directory);
|
||||
if (depth < minimumDepth) {
|
||||
selected = candidate;
|
||||
minimumDepth = depth;
|
||||
candidateCount = 1;
|
||||
continue;
|
||||
}
|
||||
if (depth !== minimumDepth) continue;
|
||||
candidateCount += 1;
|
||||
if (
|
||||
selected === undefined ||
|
||||
candidate.extracted.normalizedPath.localeCompare(
|
||||
selected.extracted.normalizedPath
|
||||
) < 0
|
||||
) {
|
||||
selected = candidate;
|
||||
}
|
||||
}
|
||||
if (candidateCount > 1) {
|
||||
diagnostics.add(
|
||||
warning(
|
||||
'onetoc2-root-ambiguous',
|
||||
'More than one top-level table of contents was found; the first safe package path is used.',
|
||||
@@ -355,12 +433,16 @@ function selectRootToc(
|
||||
)
|
||||
);
|
||||
}
|
||||
return candidates[0];
|
||||
return selected;
|
||||
}
|
||||
|
||||
export function resolveTocChildPath(
|
||||
directory: string,
|
||||
filename: string
|
||||
filename: string,
|
||||
limits: Pick<
|
||||
OnePkgLimits,
|
||||
'maxPathCharacters' | 'maxPathDepth'
|
||||
> = DEFAULT_ONEPKG_LIMITS
|
||||
): string {
|
||||
if (
|
||||
filename.length === 0 ||
|
||||
@@ -381,8 +463,8 @@ export function resolveTocChildPath(
|
||||
const normalized = filename.normalize('NFC');
|
||||
const path = directory ? `${directory}/${normalized}` : normalized;
|
||||
if (
|
||||
path.length > DEFAULT_ONEPKG_LIMITS.maxPathCharacters ||
|
||||
pathDepth(path) > DEFAULT_ONEPKG_LIMITS.maxPathDepth
|
||||
path.length > limits.maxPathCharacters ||
|
||||
pathDepth(path) > limits.maxPathDepth
|
||||
) {
|
||||
fail(
|
||||
'onetoc2-child-path-limit',
|
||||
@@ -450,6 +532,75 @@ function tocChildFailureDiagnostic(
|
||||
};
|
||||
}
|
||||
|
||||
function isFatalTocBuildError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof OnePkgError &&
|
||||
(error.diagnostic.code === 'onepkg-cancelled' ||
|
||||
error.diagnostic.code === 'onetoc2-operation-limit')
|
||||
);
|
||||
}
|
||||
|
||||
const TOC_CANCELLATION_INTERVAL = 256;
|
||||
|
||||
class TocBuildBudget {
|
||||
private operations = 0;
|
||||
private nextCancellationCheck = TOC_CANCELLATION_INTERVAL;
|
||||
|
||||
constructor(
|
||||
private readonly limits: OnePkgLimits,
|
||||
private readonly cancellation: OnePkgCancellation | undefined
|
||||
) {
|
||||
checkCancellation(cancellation);
|
||||
}
|
||||
|
||||
charge(): Promise<void> | undefined {
|
||||
this.chargeWithoutYield();
|
||||
if (this.operations < this.nextCancellationCheck) return undefined;
|
||||
this.nextCancellationCheck = this.operations + TOC_CANCELLATION_INTERVAL;
|
||||
return yieldForCancellation(this.cancellation);
|
||||
}
|
||||
|
||||
chargeWithoutYield(): void {
|
||||
if (this.operations >= this.limits.maxTocOperations) {
|
||||
fail(
|
||||
'onetoc2-operation-limit',
|
||||
`Building the notebook hierarchy exceeds ${this.limits.maxTocOperations} bounded operations.`,
|
||||
{ structure: 'ONEPKG table of contents' }
|
||||
);
|
||||
}
|
||||
this.operations += 1;
|
||||
}
|
||||
|
||||
async finish(): Promise<void> {
|
||||
await yieldForCancellation(this.cancellation);
|
||||
}
|
||||
}
|
||||
|
||||
class CappedTocDiagnostics {
|
||||
private readonly diagnostics: ParserDiagnostic[] = [];
|
||||
private truncated = false;
|
||||
|
||||
constructor(private readonly limit: number) {}
|
||||
|
||||
add(diagnostic: ParserDiagnostic): void {
|
||||
if (this.truncated) return;
|
||||
if (this.diagnostics.length < this.limit) {
|
||||
this.diagnostics.push(diagnostic);
|
||||
return;
|
||||
}
|
||||
this.truncated = true;
|
||||
this.diagnostics[this.limit - 1] = warning(
|
||||
'onetoc2-diagnostic-limit',
|
||||
`Additional notebook hierarchy diagnostics were omitted after reaching the limit of ${this.limit}.`,
|
||||
{ structure: 'ONEPKG table of contents' }
|
||||
);
|
||||
}
|
||||
|
||||
values(): ParserDiagnostic[] {
|
||||
return [...this.diagnostics];
|
||||
}
|
||||
}
|
||||
|
||||
function parentPath(path: string): string {
|
||||
const slash = path.lastIndexOf('/');
|
||||
return slash < 0 ? '' : path.slice(0, slash);
|
||||
|
||||
@@ -90,6 +90,10 @@ export interface OnePkgLimits {
|
||||
maxNameBytes: number;
|
||||
maxPathCharacters: number;
|
||||
maxPathDepth: number;
|
||||
/** Maximum indexed TOC hierarchy construction and resolution steps. */
|
||||
maxTocOperations: number;
|
||||
/** Maximum package-level diagnostics emitted while interpreting TOCs. */
|
||||
maxTocDiagnostics: number;
|
||||
maxFileBytes: number;
|
||||
maxFolderUncompressedBytes: number;
|
||||
maxTotalUncompressedBytes: number;
|
||||
@@ -111,6 +115,8 @@ export const DEFAULT_ONEPKG_LIMITS: Readonly<OnePkgLimits> = {
|
||||
maxNameBytes: 255,
|
||||
maxPathCharacters: 4_096,
|
||||
maxPathDepth: 64,
|
||||
maxTocOperations: 1_000_000,
|
||||
maxTocDiagnostics: 10_000,
|
||||
maxFileBytes: 256 * 1024 * 1024,
|
||||
maxFolderUncompressedBytes: 512 * 1024 * 1024,
|
||||
maxTotalUncompressedBytes: 512 * 1024 * 1024,
|
||||
|
||||
Reference in New Issue
Block a user