perf: bound OneNote TOC hierarchy building
This commit is contained in:
@@ -194,9 +194,10 @@ async function buildOpenedPackage(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const treeResult = buildOneNotePackageTree(
|
const treeResult = await buildOneNotePackageTree(
|
||||||
extraction.extractedEntries,
|
extraction.extractedEntries,
|
||||||
packagedSections
|
packagedSections,
|
||||||
|
options
|
||||||
);
|
);
|
||||||
diagnostics.push(...treeResult.diagnostics);
|
diagnostics.push(...treeResult.diagnostics);
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,29 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
import { resolveTocChildPath } from './toc-hierarchy.js';
|
import type { OneNotePackagedSectionDto } from '../model/dto.js';
|
||||||
|
import type {
|
||||||
|
OneNoteTableOfContents,
|
||||||
|
OneNoteTableOfContentsEntry,
|
||||||
|
} from '../parser/parse-toc.js';
|
||||||
|
import {
|
||||||
|
DESKTOP_REVISION_STORE_FORMAT,
|
||||||
|
ONE_SECTION_FILE_TYPE,
|
||||||
|
} from '../parser/detect-format.js';
|
||||||
|
import type { OnePkgError } from './errors.js';
|
||||||
|
import {
|
||||||
|
buildOneNotePackageTree,
|
||||||
|
resolveTocChildPath,
|
||||||
|
} from './toc-hierarchy.js';
|
||||||
|
import type { ExtractedCabFile } from './types.js';
|
||||||
|
|
||||||
|
const parseTocMock = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
|
vi.mock('../parser/parse-toc.js', async (importOriginal) => ({
|
||||||
|
...(await importOriginal<typeof import('../parser/parse-toc.js')>()),
|
||||||
|
parseOneNoteTableOfContents: parseTocMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
beforeEach(() => parseTocMock.mockReset());
|
||||||
|
|
||||||
describe('ONEPKG TOC child paths', () => {
|
describe('ONEPKG TOC child paths', () => {
|
||||||
it('joins a single NFC-normalized child name within its section group', () => {
|
it('joins a single NFC-normalized child name within its section group', () => {
|
||||||
@@ -18,3 +41,244 @@ describe('ONEPKG TOC child paths', () => {
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('ONEPKG TOC hierarchy budgets and indexes', () => {
|
||||||
|
it('resolves identity fallbacks in authored order with near-linear section lookups', async () => {
|
||||||
|
const count = 400;
|
||||||
|
const pathReads = { count: 0 };
|
||||||
|
const entries: OneNoteTableOfContentsEntry[] = [];
|
||||||
|
const extracted: ExtractedCabFile[] = [tocFile()];
|
||||||
|
const sections: OneNotePackagedSectionDto[] = [];
|
||||||
|
for (let index = 0; index < count; index += 1) {
|
||||||
|
const identity = indexedGuid(index + 1);
|
||||||
|
const path = `Section-${index.toString().padStart(4, '0')}.one`;
|
||||||
|
extracted.push(sectionFile(path, identity, index + 1));
|
||||||
|
sections.push(sectionWithObservedPath(path, pathReads));
|
||||||
|
entries.push({
|
||||||
|
filename: `Alias-${index.toString().padStart(4, '0')}.one`,
|
||||||
|
fileIdentity: identity,
|
||||||
|
orderingId: index,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
parseTocMock.mockReturnValue(toc(entries));
|
||||||
|
let yields = 0;
|
||||||
|
|
||||||
|
const result = await buildOneNotePackageTree(extracted, sections, {
|
||||||
|
limits: { maxTocOperations: count * 8 + 100 },
|
||||||
|
cancellation: {
|
||||||
|
yield: () => {
|
||||||
|
yields += 1;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.tree.nodes).toHaveLength(count);
|
||||||
|
expect(
|
||||||
|
result.tree.nodes.map((node) =>
|
||||||
|
node.kind === 'section' ? node.sectionId : `group:${node.id}`
|
||||||
|
)
|
||||||
|
).toEqual(sections.map(({ id }) => id));
|
||||||
|
expect(pathReads.count).toBeLessThanOrEqual(count * 5);
|
||||||
|
expect(yields).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps ambiguous identities unresolved and fallback sections sorted', async () => {
|
||||||
|
const identity = indexedGuid(42);
|
||||||
|
parseTocMock.mockReturnValue(
|
||||||
|
toc([{ filename: 'Alias.one', fileIdentity: identity, orderingId: 1 }])
|
||||||
|
);
|
||||||
|
const sections = [section('Z.one'), section('A.one')];
|
||||||
|
|
||||||
|
const result = await buildOneNotePackageTree(
|
||||||
|
[
|
||||||
|
tocFile(),
|
||||||
|
sectionFile('Z.one', identity, 1),
|
||||||
|
sectionFile('A.one', identity, 2),
|
||||||
|
],
|
||||||
|
sections
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
result.tree.nodes.map((node) =>
|
||||||
|
node.kind === 'section' ? node.sectionId : `group:${node.id}`
|
||||||
|
)
|
||||||
|
).toEqual(['A.one', 'Z.one']);
|
||||||
|
expect(result.diagnostics.map(({ code }) => code)).toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
'onetoc2-entry-unresolved',
|
||||||
|
'onetoc2-unlisted-sections',
|
||||||
|
])
|
||||||
|
);
|
||||||
|
expect(result.diagnostics).not.toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
expect.objectContaining({
|
||||||
|
code: 'onetoc2-section-filename-mismatch',
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('caps adversarial unresolved-entry diagnostics with an explicit marker', async () => {
|
||||||
|
parseTocMock.mockReturnValue(
|
||||||
|
toc(
|
||||||
|
Array.from({ length: 50 }, (_, index) => ({
|
||||||
|
filename: `Missing-${index}.one`,
|
||||||
|
fileIdentity: indexedGuid(index + 1),
|
||||||
|
orderingId: index,
|
||||||
|
}))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await buildOneNotePackageTree([tocFile()], [], {
|
||||||
|
limits: { maxTocDiagnostics: 5, maxTocOperations: 1_000 },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.diagnostics).toHaveLength(5);
|
||||||
|
expect(result.diagnostics.slice(0, 4).map(({ code }) => code)).toEqual([
|
||||||
|
'onetoc2-entry-unresolved',
|
||||||
|
'onetoc2-entry-unresolved',
|
||||||
|
'onetoc2-entry-unresolved',
|
||||||
|
'onetoc2-entry-unresolved',
|
||||||
|
]);
|
||||||
|
expect(result.diagnostics[4]?.code).toBe('onetoc2-diagnostic-limit');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fails closed at the hierarchy operation limit', async () => {
|
||||||
|
parseTocMock.mockReturnValue(
|
||||||
|
toc(
|
||||||
|
Array.from({ length: 20 }, (_, index) => ({
|
||||||
|
filename: `Missing-${index}.one`,
|
||||||
|
fileIdentity: indexedGuid(index + 1),
|
||||||
|
orderingId: index,
|
||||||
|
}))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
buildOneNotePackageTree([tocFile()], [], {
|
||||||
|
limits: { maxTocOperations: 10 },
|
||||||
|
})
|
||||||
|
).rejects.toEqual(
|
||||||
|
expect.objectContaining<Partial<OnePkgError>>({
|
||||||
|
diagnostic: expect.objectContaining({
|
||||||
|
code: 'onetoc2-operation-limit',
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('yields and observes cancellation during a large hierarchy build', async () => {
|
||||||
|
parseTocMock.mockReturnValue(
|
||||||
|
toc(
|
||||||
|
Array.from({ length: 1_000 }, (_, index) => ({
|
||||||
|
filename: `Missing-${index}.one`,
|
||||||
|
fileIdentity: indexedGuid(index + 1),
|
||||||
|
orderingId: index,
|
||||||
|
}))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const controller = new AbortController();
|
||||||
|
let yields = 0;
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
buildOneNotePackageTree([tocFile()], [], {
|
||||||
|
cancellation: {
|
||||||
|
signal: controller.signal,
|
||||||
|
yield: () => {
|
||||||
|
yields += 1;
|
||||||
|
controller.abort();
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
).rejects.toEqual(
|
||||||
|
expect.objectContaining<Partial<OnePkgError>>({
|
||||||
|
diagnostic: expect.objectContaining({ code: 'onepkg-cancelled' }),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(yields).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
function toc(entries: OneNoteTableOfContentsEntry[]): OneNoteTableOfContents {
|
||||||
|
return {
|
||||||
|
sourceName: 'Open Notebook.onetoc2',
|
||||||
|
fileIdentity: indexedGuid(0),
|
||||||
|
entries,
|
||||||
|
diagnostics: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function tocFile(): ExtractedCabFile {
|
||||||
|
return extractedFile('Open Notebook.onetoc2', Uint8Array.of(1), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sectionFile(
|
||||||
|
path: string,
|
||||||
|
identity: string,
|
||||||
|
index: number
|
||||||
|
): ExtractedCabFile {
|
||||||
|
const bytes = new Uint8Array(64);
|
||||||
|
writeGuid(bytes, 0, ONE_SECTION_FILE_TYPE);
|
||||||
|
writeGuid(bytes, 16, identity);
|
||||||
|
writeGuid(bytes, 48, DESKTOP_REVISION_STORE_FORMAT);
|
||||||
|
return extractedFile(path, bytes, index);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractedFile(
|
||||||
|
path: string,
|
||||||
|
bytes: Uint8Array,
|
||||||
|
index: number
|
||||||
|
): ExtractedCabFile {
|
||||||
|
return {
|
||||||
|
index,
|
||||||
|
path,
|
||||||
|
normalizedPath: path,
|
||||||
|
folderIndex: 0,
|
||||||
|
uncompressedOffset: 0,
|
||||||
|
uncompressedSize: bytes.byteLength,
|
||||||
|
attributes: 0,
|
||||||
|
compression: { kind: 'none', raw: 0, label: 'None' },
|
||||||
|
bytes,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function section(path: string): OneNotePackagedSectionDto {
|
||||||
|
return {
|
||||||
|
id: path,
|
||||||
|
path,
|
||||||
|
displayName: path.slice(0, -4),
|
||||||
|
parseStatus: 'not-requested',
|
||||||
|
diagnostics: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sectionWithObservedPath(
|
||||||
|
path: string,
|
||||||
|
reads: { count: number }
|
||||||
|
): OneNotePackagedSectionDto {
|
||||||
|
const result = section(path);
|
||||||
|
Object.defineProperty(result, 'path', {
|
||||||
|
enumerable: true,
|
||||||
|
get: () => {
|
||||||
|
reads.count += 1;
|
||||||
|
return path;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function indexedGuid(index: number): string {
|
||||||
|
const plain = index.toString(16).toUpperCase().padStart(32, '0');
|
||||||
|
return `{${plain.slice(0, 8)}-${plain.slice(8, 12)}-${plain.slice(12, 16)}-${plain.slice(16, 20)}-${plain.slice(20)}}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeGuid(target: Uint8Array, offset: number, guid: string): void {
|
||||||
|
const plain = guid.replace(/[{}-]/gu, '');
|
||||||
|
const canonical = Array.from({ length: 16 }, (_, index) =>
|
||||||
|
Number.parseInt(plain.slice(index * 2, index * 2 + 2), 16)
|
||||||
|
);
|
||||||
|
const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15];
|
||||||
|
for (let index = 0; index < order.length; index += 1) {
|
||||||
|
target[offset + order[index]!] = canonical[index]!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,9 +16,16 @@ import {
|
|||||||
type OneNoteTableOfContents,
|
type OneNoteTableOfContents,
|
||||||
type OneNoteTableOfContentsEntry,
|
type OneNoteTableOfContentsEntry,
|
||||||
} from '../parser/parse-toc.js';
|
} from '../parser/parse-toc.js';
|
||||||
|
import { checkCancellation, yieldForCancellation } from './cancellation.js';
|
||||||
import { fail, OnePkgError, warning } from './errors.js';
|
import { fail, OnePkgError, warning } from './errors.js';
|
||||||
import type { ExtractedCabFile } from './types.js';
|
import {
|
||||||
import { DEFAULT_ONEPKG_LIMITS } from './types.js';
|
DEFAULT_ONEPKG_LIMITS,
|
||||||
|
resolveLimits,
|
||||||
|
type CabOpenOptions,
|
||||||
|
type ExtractedCabFile,
|
||||||
|
type OnePkgCancellation,
|
||||||
|
type OnePkgLimits,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
interface ParsedTocEntry {
|
interface ParsedTocEntry {
|
||||||
extracted: ExtractedCabFile;
|
extracted: ExtractedCabFile;
|
||||||
@@ -35,18 +42,24 @@ export interface OneNotePackageTreeResult {
|
|||||||
* Interpret package TOCs and join them to extracted sections without ever
|
* Interpret package TOCs and join them to extracted sections without ever
|
||||||
* materializing an archive path on the host filesystem.
|
* materializing an archive path on the host filesystem.
|
||||||
*/
|
*/
|
||||||
export function buildOneNotePackageTree(
|
export async function buildOneNotePackageTree(
|
||||||
extractedEntries: ExtractedCabFile[],
|
extractedEntries: ExtractedCabFile[],
|
||||||
sections: OneNotePackagedSectionDto[]
|
sections: OneNotePackagedSectionDto[],
|
||||||
): OneNotePackageTreeResult {
|
options: CabOpenOptions = {}
|
||||||
const diagnostics: ParserDiagnostic[] = [];
|
): Promise<OneNotePackageTreeResult> {
|
||||||
|
const limits = resolveLimits(options.limits);
|
||||||
|
const budget = new TocBuildBudget(limits, options.cancellation);
|
||||||
|
const diagnostics = new CappedTocDiagnostics(limits.maxTocDiagnostics);
|
||||||
const parsedTocs: ParsedTocEntry[] = [];
|
const parsedTocs: ParsedTocEntry[] = [];
|
||||||
|
|
||||||
for (const extracted of extractedEntries) {
|
for (const extracted of extractedEntries) {
|
||||||
|
const checkpoint = budget.charge();
|
||||||
|
if (checkpoint) await checkpoint;
|
||||||
if (!extracted.normalizedPath.toLowerCase().endsWith('.onetoc2')) continue;
|
if (!extracted.normalizedPath.toLowerCase().endsWith('.onetoc2')) continue;
|
||||||
try {
|
try {
|
||||||
const toc = parseOneNoteTableOfContents(extracted.bytes, {
|
const toc = parseOneNoteTableOfContents(extracted.bytes, {
|
||||||
sourceName: extracted.normalizedPath,
|
sourceName: extracted.normalizedPath,
|
||||||
|
limits: { maxDiagnostics: limits.maxTocDiagnostics },
|
||||||
});
|
});
|
||||||
parsedTocs.push({
|
parsedTocs.push({
|
||||||
extracted,
|
extracted,
|
||||||
@@ -54,8 +67,9 @@ export function buildOneNotePackageTree(
|
|||||||
directory: parentPath(extracted.normalizedPath),
|
directory: parentPath(extracted.normalizedPath),
|
||||||
});
|
});
|
||||||
for (const diagnostic of toc.diagnostics) {
|
for (const diagnostic of toc.diagnostics) {
|
||||||
if (diagnostics.length >= DEFAULT_ONEPKG_LIMITS.maxFiles) break;
|
const diagnosticCheckpoint = budget.charge();
|
||||||
diagnostics.push({
|
if (diagnosticCheckpoint) await diagnosticCheckpoint;
|
||||||
|
diagnostics.add({
|
||||||
...diagnostic,
|
...diagnostic,
|
||||||
structure: diagnostic.structure
|
structure: diagnostic.structure
|
||||||
? `${extracted.normalizedPath} · ${diagnostic.structure}`
|
? `${extracted.normalizedPath} · ${diagnostic.structure}`
|
||||||
@@ -63,29 +77,63 @@ export function buildOneNotePackageTree(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
diagnostics.push(tocFailureDiagnostic(error, extracted.normalizedPath));
|
if (isFatalTocBuildError(error)) throw error;
|
||||||
|
diagnostics.add(tocFailureDiagnostic(error, extracted.normalizedPath));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const root = selectRootToc(parsedTocs, diagnostics);
|
const root = await selectRootToc(parsedTocs, diagnostics, budget);
|
||||||
const sectionByPath = new Map(
|
const sectionByPath = new Map<string, OneNotePackagedSectionDto>();
|
||||||
sections.map((section) => [pathKey(section.path), section])
|
for (const section of sections) {
|
||||||
);
|
const checkpoint = budget.charge();
|
||||||
const extractedSectionByPath = new Map(
|
if (checkpoint) await checkpoint;
|
||||||
extractedEntries
|
sectionByPath.set(pathKey(section.path), section);
|
||||||
.filter((entry) => entry.normalizedPath.toLowerCase().endsWith('.one'))
|
}
|
||||||
.map((entry) => [pathKey(entry.normalizedPath), entry])
|
const extractedSectionByPath = new Map<string, ExtractedCabFile>();
|
||||||
);
|
for (const entry of extractedEntries) {
|
||||||
|
const checkpoint = budget.charge();
|
||||||
|
if (checkpoint) await checkpoint;
|
||||||
|
if (!entry.normalizedPath.toLowerCase().endsWith('.one')) continue;
|
||||||
|
extractedSectionByPath.set(pathKey(entry.normalizedPath), entry);
|
||||||
|
}
|
||||||
const sectionIdentityByPath = new Map<string, string>();
|
const sectionIdentityByPath = new Map<string, string>();
|
||||||
for (const [key, entry] of extractedSectionByPath) {
|
for (const [key, entry] of extractedSectionByPath) {
|
||||||
|
const checkpoint = budget.charge();
|
||||||
|
if (checkpoint) await checkpoint;
|
||||||
const identity = readSectionIdentity(entry.bytes);
|
const identity = readSectionIdentity(entry.bytes);
|
||||||
if (identity) sectionIdentityByPath.set(key, identity);
|
if (identity) sectionIdentityByPath.set(key, identity);
|
||||||
}
|
}
|
||||||
|
const sectionByDirectoryAndIdentity = new Map<
|
||||||
|
string,
|
||||||
|
Map<string, IdentitySectionMatch>
|
||||||
|
>();
|
||||||
|
for (const section of sections) {
|
||||||
|
const checkpoint = budget.charge();
|
||||||
|
if (checkpoint) await checkpoint;
|
||||||
|
const sectionPathKey = pathKey(section.path);
|
||||||
|
const identity = sectionIdentityByPath.get(sectionPathKey);
|
||||||
|
if (!identity) continue;
|
||||||
|
const directoryKey = pathKey(parentPath(section.path));
|
||||||
|
let byIdentity = sectionByDirectoryAndIdentity.get(directoryKey);
|
||||||
|
if (!byIdentity) {
|
||||||
|
byIdentity = new Map();
|
||||||
|
sectionByDirectoryAndIdentity.set(directoryKey, byIdentity);
|
||||||
|
}
|
||||||
|
const existing = byIdentity.get(identity);
|
||||||
|
byIdentity.set(
|
||||||
|
identity,
|
||||||
|
existing
|
||||||
|
? { section: existing.section, ambiguous: true }
|
||||||
|
: { section, ambiguous: false }
|
||||||
|
);
|
||||||
|
}
|
||||||
const tocByDirectory = new Map<string, ParsedTocEntry>();
|
const tocByDirectory = new Map<string, ParsedTocEntry>();
|
||||||
for (const parsed of parsedTocs) {
|
for (const parsed of parsedTocs) {
|
||||||
|
const checkpoint = budget.charge();
|
||||||
|
if (checkpoint) await checkpoint;
|
||||||
const key = pathKey(parsed.directory);
|
const key = pathKey(parsed.directory);
|
||||||
if (tocByDirectory.has(key)) {
|
if (tocByDirectory.has(key)) {
|
||||||
diagnostics.push(
|
diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-directory-ambiguous',
|
'onetoc2-directory-ambiguous',
|
||||||
`More than one readable .onetoc2 exists in ${parsed.directory || 'the package root'}; the first one is used.`,
|
`More than one readable .onetoc2 exists in ${parsed.directory || 'the package root'}; the first one is used.`,
|
||||||
@@ -100,28 +148,36 @@ export function buildOneNotePackageTree(
|
|||||||
const usedSectionIds = new Set<string>();
|
const usedSectionIds = new Set<string>();
|
||||||
let nodes: OneNoteNotebookNodeDto[] = [];
|
let nodes: OneNoteNotebookNodeDto[] = [];
|
||||||
if (root) {
|
if (root) {
|
||||||
nodes = buildTocNodes({
|
nodes = await buildTocNodes({
|
||||||
parsed: root,
|
parsed: root,
|
||||||
tocByDirectory,
|
tocByDirectory,
|
||||||
sectionByPath,
|
sectionByPath,
|
||||||
sectionIdentityByPath,
|
sectionIdentityByPath,
|
||||||
sections,
|
sectionByDirectoryAndIdentity,
|
||||||
usedSectionIds,
|
usedSectionIds,
|
||||||
diagnostics,
|
diagnostics,
|
||||||
|
limits,
|
||||||
|
budget,
|
||||||
activeTocs: new Set(),
|
activeTocs: new Set(),
|
||||||
usedTocs: new Set(),
|
usedTocs: new Set(),
|
||||||
depth: 0,
|
depth: 0,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const unlisted = sections
|
const unlisted: OneNotePackagedSectionDto[] = [];
|
||||||
.filter(
|
for (const section of sections) {
|
||||||
(section) =>
|
const checkpoint = budget.charge();
|
||||||
!usedSectionIds.has(section.id) && !isRecycleBinPath(section.path)
|
if (checkpoint) await checkpoint;
|
||||||
)
|
if (!usedSectionIds.has(section.id) && !isRecycleBinPath(section.path)) {
|
||||||
.sort((left, right) => left.path.localeCompare(right.path));
|
unlisted.push(section);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unlisted.sort((left, right) => {
|
||||||
|
budget.chargeWithoutYield();
|
||||||
|
return left.path.localeCompare(right.path);
|
||||||
|
});
|
||||||
if (unlisted.length > 0 && root) {
|
if (unlisted.length > 0 && root) {
|
||||||
diagnostics.push(
|
diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-unlisted-sections',
|
'onetoc2-unlisted-sections',
|
||||||
`${unlisted.length} package section${unlisted.length === 1 ? '' : 's'} were not reachable from the parsed TOC and were appended to navigation.`,
|
`${unlisted.length} package section${unlisted.length === 1 ? '' : 's'} were not reachable from the parsed TOC and were appended to navigation.`,
|
||||||
@@ -130,6 +186,7 @@ export function buildOneNotePackageTree(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
nodes.push(...unlisted.map(fallbackSectionNode));
|
nodes.push(...unlisted.map(fallbackSectionNode));
|
||||||
|
await budget.finish();
|
||||||
|
|
||||||
return {
|
return {
|
||||||
tree: {
|
tree: {
|
||||||
@@ -138,28 +195,38 @@ export function buildOneNotePackageTree(
|
|||||||
color: root?.toc.color,
|
color: root?.toc.color,
|
||||||
nodes,
|
nodes,
|
||||||
},
|
},
|
||||||
diagnostics,
|
diagnostics: diagnostics.values(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface IdentitySectionMatch {
|
||||||
|
section: OneNotePackagedSectionDto;
|
||||||
|
ambiguous: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
interface BuildTocNodesContext {
|
interface BuildTocNodesContext {
|
||||||
parsed: ParsedTocEntry;
|
parsed: ParsedTocEntry;
|
||||||
tocByDirectory: ReadonlyMap<string, ParsedTocEntry>;
|
tocByDirectory: ReadonlyMap<string, ParsedTocEntry>;
|
||||||
sectionByPath: ReadonlyMap<string, OneNotePackagedSectionDto>;
|
sectionByPath: ReadonlyMap<string, OneNotePackagedSectionDto>;
|
||||||
sectionIdentityByPath: ReadonlyMap<string, string>;
|
sectionIdentityByPath: ReadonlyMap<string, string>;
|
||||||
sections: OneNotePackagedSectionDto[];
|
sectionByDirectoryAndIdentity: ReadonlyMap<
|
||||||
|
string,
|
||||||
|
ReadonlyMap<string, IdentitySectionMatch>
|
||||||
|
>;
|
||||||
usedSectionIds: Set<string>;
|
usedSectionIds: Set<string>;
|
||||||
diagnostics: ParserDiagnostic[];
|
diagnostics: CappedTocDiagnostics;
|
||||||
|
limits: OnePkgLimits;
|
||||||
|
budget: TocBuildBudget;
|
||||||
activeTocs: Set<string>;
|
activeTocs: Set<string>;
|
||||||
usedTocs: Set<string>;
|
usedTocs: Set<string>;
|
||||||
depth: number;
|
depth: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildTocNodes(
|
async function buildTocNodes(
|
||||||
context: BuildTocNodesContext
|
context: BuildTocNodesContext
|
||||||
): OneNoteNotebookNodeDto[] {
|
): Promise<OneNoteNotebookNodeDto[]> {
|
||||||
if (context.depth > DEFAULT_ONEPKG_LIMITS.maxPathDepth) {
|
if (context.depth > context.limits.maxPathDepth) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-tree-depth-limit',
|
'onetoc2-tree-depth-limit',
|
||||||
'Notebook section-group nesting exceeds the package path-depth limit.',
|
'Notebook section-group nesting exceeds the package path-depth limit.',
|
||||||
@@ -170,7 +237,7 @@ function buildTocNodes(
|
|||||||
}
|
}
|
||||||
const tocKey = pathKey(context.parsed.extracted.normalizedPath);
|
const tocKey = pathKey(context.parsed.extracted.normalizedPath);
|
||||||
if (context.activeTocs.has(tocKey)) {
|
if (context.activeTocs.has(tocKey)) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-tree-cycle',
|
'onetoc2-tree-cycle',
|
||||||
'A cycle between notebook table-of-contents files was ignored.',
|
'A cycle between notebook table-of-contents files was ignored.',
|
||||||
@@ -180,7 +247,7 @@ function buildTocNodes(
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
if (context.usedTocs.has(tocKey)) {
|
if (context.usedTocs.has(tocKey)) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-tree-duplicate',
|
'onetoc2-tree-duplicate',
|
||||||
'A repeated notebook section-group reference was ignored.',
|
'A repeated notebook section-group reference was ignored.',
|
||||||
@@ -195,16 +262,19 @@ function buildTocNodes(
|
|||||||
try {
|
try {
|
||||||
const result: OneNoteNotebookNodeDto[] = [];
|
const result: OneNoteNotebookNodeDto[] = [];
|
||||||
for (const entry of context.parsed.toc.entries) {
|
for (const entry of context.parsed.toc.entries) {
|
||||||
|
const checkpoint = context.budget.charge();
|
||||||
|
if (checkpoint) await checkpoint;
|
||||||
if (isRecycleBinName(entry.filename)) continue;
|
if (isRecycleBinName(entry.filename)) continue;
|
||||||
try {
|
try {
|
||||||
const childPath = resolveTocChildPath(
|
const childPath = resolveTocChildPath(
|
||||||
context.parsed.directory,
|
context.parsed.directory,
|
||||||
entry.filename
|
entry.filename,
|
||||||
|
context.limits
|
||||||
);
|
);
|
||||||
const section = resolveSection(context, childPath, entry);
|
const section = resolveSection(context, childPath, entry);
|
||||||
if (section) {
|
if (section) {
|
||||||
if (context.usedSectionIds.has(section.id)) {
|
if (context.usedSectionIds.has(section.id)) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-section-duplicate',
|
'onetoc2-section-duplicate',
|
||||||
`Repeated TOC reference to ${entry.filename} was ignored.`,
|
`Repeated TOC reference to ${entry.filename} was ignored.`,
|
||||||
@@ -229,7 +299,7 @@ function buildTocNodes(
|
|||||||
if (childToc) {
|
if (childToc) {
|
||||||
const childTocKey = pathKey(childToc.extracted.normalizedPath);
|
const childTocKey = pathKey(childToc.extracted.normalizedPath);
|
||||||
if (context.activeTocs.has(childTocKey)) {
|
if (context.activeTocs.has(childTocKey)) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-tree-cycle',
|
'onetoc2-tree-cycle',
|
||||||
'A cycle between notebook table-of-contents files was ignored.',
|
'A cycle between notebook table-of-contents files was ignored.',
|
||||||
@@ -239,7 +309,7 @@ function buildTocNodes(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (context.usedTocs.has(childTocKey)) {
|
if (context.usedTocs.has(childTocKey)) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-tree-duplicate',
|
'onetoc2-tree-duplicate',
|
||||||
'A repeated notebook section-group reference was ignored.',
|
'A repeated notebook section-group reference was ignored.',
|
||||||
@@ -254,7 +324,7 @@ function buildTocNodes(
|
|||||||
displayName: leafName(childPath),
|
displayName: leafName(childPath),
|
||||||
orderingId: entry.orderingId,
|
orderingId: entry.orderingId,
|
||||||
color: entry.color,
|
color: entry.color,
|
||||||
children: buildTocNodes({
|
children: await buildTocNodes({
|
||||||
...context,
|
...context,
|
||||||
parsed: childToc,
|
parsed: childToc,
|
||||||
depth: context.depth + 1,
|
depth: context.depth + 1,
|
||||||
@@ -263,7 +333,7 @@ function buildTocNodes(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-entry-unresolved',
|
'onetoc2-entry-unresolved',
|
||||||
`TOC entry ${entry.filename} has no matching section or section group in the package.`,
|
`TOC entry ${entry.filename} has no matching section or section group in the package.`,
|
||||||
@@ -271,7 +341,8 @@ function buildTocNodes(
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
context.diagnostics.push(
|
if (isFatalTocBuildError(error)) throw error;
|
||||||
|
context.diagnostics.add(
|
||||||
tocChildFailureDiagnostic(
|
tocChildFailureDiagnostic(
|
||||||
error,
|
error,
|
||||||
context.parsed.extracted.normalizedPath,
|
context.parsed.extracted.normalizedPath,
|
||||||
@@ -296,7 +367,7 @@ function resolveSection(
|
|||||||
if (exact) {
|
if (exact) {
|
||||||
const identity = context.sectionIdentityByPath.get(expectedKey);
|
const identity = context.sectionIdentityByPath.get(expectedKey);
|
||||||
if (identity && identity !== tocEntry.fileIdentity) {
|
if (identity && identity !== tocEntry.fileIdentity) {
|
||||||
context.diagnostics.push(
|
context.diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-section-identity-mismatch',
|
'onetoc2-section-identity-mismatch',
|
||||||
`TOC identity for ${tocEntry.filename} does not match the section header; the package filename takes precedence.`,
|
`TOC identity for ${tocEntry.filename} does not match the section header; the package filename takes precedence.`,
|
||||||
@@ -307,47 +378,54 @@ function resolveSection(
|
|||||||
return exact;
|
return exact;
|
||||||
}
|
}
|
||||||
|
|
||||||
const directory = parentPath(childPath);
|
const match = context.sectionByDirectoryAndIdentity
|
||||||
const identityMatches = context.sections.filter(
|
.get(pathKey(parentPath(childPath)))
|
||||||
(section) =>
|
?.get(tocEntry.fileIdentity);
|
||||||
pathKey(parentPath(section.path)) === pathKey(directory) &&
|
if (match && !match.ambiguous) {
|
||||||
context.sectionIdentityByPath.get(pathKey(section.path)) ===
|
context.diagnostics.add(
|
||||||
tocEntry.fileIdentity
|
|
||||||
);
|
|
||||||
if (identityMatches.length === 1) {
|
|
||||||
const match = identityMatches[0]!;
|
|
||||||
context.diagnostics.push(
|
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-section-filename-mismatch',
|
'onetoc2-section-filename-mismatch',
|
||||||
`TOC filename ${tocEntry.filename} was resolved to package section ${leafName(match.path)} by FileIdentityGuid.`,
|
`TOC filename ${tocEntry.filename} was resolved to package section ${leafName(match.section.path)} by FileIdentityGuid.`,
|
||||||
{ structure: context.parsed.extracted.normalizedPath }
|
{ structure: context.parsed.extracted.normalizedPath }
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
return match;
|
return match.section;
|
||||||
}
|
}
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
function selectRootToc(
|
async function selectRootToc(
|
||||||
parsedTocs: ParsedTocEntry[],
|
parsedTocs: ParsedTocEntry[],
|
||||||
diagnostics: ParserDiagnostic[]
|
diagnostics: CappedTocDiagnostics,
|
||||||
): ParsedTocEntry | undefined {
|
budget: TocBuildBudget
|
||||||
|
): Promise<ParsedTocEntry | undefined> {
|
||||||
if (parsedTocs.length === 0) return undefined;
|
if (parsedTocs.length === 0) return undefined;
|
||||||
const sorted = [...parsedTocs].sort((left, right) => {
|
let selected: ParsedTocEntry | undefined;
|
||||||
const depth = pathDepth(left.directory) - pathDepth(right.directory);
|
let minimumDepth = Number.POSITIVE_INFINITY;
|
||||||
return (
|
let candidateCount = 0;
|
||||||
depth ||
|
for (const candidate of parsedTocs) {
|
||||||
left.extracted.normalizedPath.localeCompare(
|
const checkpoint = budget.charge();
|
||||||
right.extracted.normalizedPath
|
if (checkpoint) await checkpoint;
|
||||||
)
|
const depth = pathDepth(candidate.directory);
|
||||||
);
|
if (depth < minimumDepth) {
|
||||||
});
|
selected = candidate;
|
||||||
const minimumDepth = pathDepth(sorted[0]!.directory);
|
minimumDepth = depth;
|
||||||
const candidates = sorted.filter(
|
candidateCount = 1;
|
||||||
(candidate) => pathDepth(candidate.directory) === minimumDepth
|
continue;
|
||||||
);
|
}
|
||||||
if (candidates.length > 1) {
|
if (depth !== minimumDepth) continue;
|
||||||
diagnostics.push(
|
candidateCount += 1;
|
||||||
|
if (
|
||||||
|
selected === undefined ||
|
||||||
|
candidate.extracted.normalizedPath.localeCompare(
|
||||||
|
selected.extracted.normalizedPath
|
||||||
|
) < 0
|
||||||
|
) {
|
||||||
|
selected = candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (candidateCount > 1) {
|
||||||
|
diagnostics.add(
|
||||||
warning(
|
warning(
|
||||||
'onetoc2-root-ambiguous',
|
'onetoc2-root-ambiguous',
|
||||||
'More than one top-level table of contents was found; the first safe package path is used.',
|
'More than one top-level table of contents was found; the first safe package path is used.',
|
||||||
@@ -355,12 +433,16 @@ function selectRootToc(
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return candidates[0];
|
return selected;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTocChildPath(
|
export function resolveTocChildPath(
|
||||||
directory: string,
|
directory: string,
|
||||||
filename: string
|
filename: string,
|
||||||
|
limits: Pick<
|
||||||
|
OnePkgLimits,
|
||||||
|
'maxPathCharacters' | 'maxPathDepth'
|
||||||
|
> = DEFAULT_ONEPKG_LIMITS
|
||||||
): string {
|
): string {
|
||||||
if (
|
if (
|
||||||
filename.length === 0 ||
|
filename.length === 0 ||
|
||||||
@@ -381,8 +463,8 @@ export function resolveTocChildPath(
|
|||||||
const normalized = filename.normalize('NFC');
|
const normalized = filename.normalize('NFC');
|
||||||
const path = directory ? `${directory}/${normalized}` : normalized;
|
const path = directory ? `${directory}/${normalized}` : normalized;
|
||||||
if (
|
if (
|
||||||
path.length > DEFAULT_ONEPKG_LIMITS.maxPathCharacters ||
|
path.length > limits.maxPathCharacters ||
|
||||||
pathDepth(path) > DEFAULT_ONEPKG_LIMITS.maxPathDepth
|
pathDepth(path) > limits.maxPathDepth
|
||||||
) {
|
) {
|
||||||
fail(
|
fail(
|
||||||
'onetoc2-child-path-limit',
|
'onetoc2-child-path-limit',
|
||||||
@@ -450,6 +532,75 @@ function tocChildFailureDiagnostic(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isFatalTocBuildError(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
error instanceof OnePkgError &&
|
||||||
|
(error.diagnostic.code === 'onepkg-cancelled' ||
|
||||||
|
error.diagnostic.code === 'onetoc2-operation-limit')
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const TOC_CANCELLATION_INTERVAL = 256;
|
||||||
|
|
||||||
|
class TocBuildBudget {
|
||||||
|
private operations = 0;
|
||||||
|
private nextCancellationCheck = TOC_CANCELLATION_INTERVAL;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly limits: OnePkgLimits,
|
||||||
|
private readonly cancellation: OnePkgCancellation | undefined
|
||||||
|
) {
|
||||||
|
checkCancellation(cancellation);
|
||||||
|
}
|
||||||
|
|
||||||
|
charge(): Promise<void> | undefined {
|
||||||
|
this.chargeWithoutYield();
|
||||||
|
if (this.operations < this.nextCancellationCheck) return undefined;
|
||||||
|
this.nextCancellationCheck = this.operations + TOC_CANCELLATION_INTERVAL;
|
||||||
|
return yieldForCancellation(this.cancellation);
|
||||||
|
}
|
||||||
|
|
||||||
|
chargeWithoutYield(): void {
|
||||||
|
if (this.operations >= this.limits.maxTocOperations) {
|
||||||
|
fail(
|
||||||
|
'onetoc2-operation-limit',
|
||||||
|
`Building the notebook hierarchy exceeds ${this.limits.maxTocOperations} bounded operations.`,
|
||||||
|
{ structure: 'ONEPKG table of contents' }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.operations += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
async finish(): Promise<void> {
|
||||||
|
await yieldForCancellation(this.cancellation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class CappedTocDiagnostics {
|
||||||
|
private readonly diagnostics: ParserDiagnostic[] = [];
|
||||||
|
private truncated = false;
|
||||||
|
|
||||||
|
constructor(private readonly limit: number) {}
|
||||||
|
|
||||||
|
add(diagnostic: ParserDiagnostic): void {
|
||||||
|
if (this.truncated) return;
|
||||||
|
if (this.diagnostics.length < this.limit) {
|
||||||
|
this.diagnostics.push(diagnostic);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.truncated = true;
|
||||||
|
this.diagnostics[this.limit - 1] = warning(
|
||||||
|
'onetoc2-diagnostic-limit',
|
||||||
|
`Additional notebook hierarchy diagnostics were omitted after reaching the limit of ${this.limit}.`,
|
||||||
|
{ structure: 'ONEPKG table of contents' }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
values(): ParserDiagnostic[] {
|
||||||
|
return [...this.diagnostics];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function parentPath(path: string): string {
|
function parentPath(path: string): string {
|
||||||
const slash = path.lastIndexOf('/');
|
const slash = path.lastIndexOf('/');
|
||||||
return slash < 0 ? '' : path.slice(0, slash);
|
return slash < 0 ? '' : path.slice(0, slash);
|
||||||
|
|||||||
@@ -90,6 +90,10 @@ export interface OnePkgLimits {
|
|||||||
maxNameBytes: number;
|
maxNameBytes: number;
|
||||||
maxPathCharacters: number;
|
maxPathCharacters: number;
|
||||||
maxPathDepth: number;
|
maxPathDepth: number;
|
||||||
|
/** Maximum indexed TOC hierarchy construction and resolution steps. */
|
||||||
|
maxTocOperations: number;
|
||||||
|
/** Maximum package-level diagnostics emitted while interpreting TOCs. */
|
||||||
|
maxTocDiagnostics: number;
|
||||||
maxFileBytes: number;
|
maxFileBytes: number;
|
||||||
maxFolderUncompressedBytes: number;
|
maxFolderUncompressedBytes: number;
|
||||||
maxTotalUncompressedBytes: number;
|
maxTotalUncompressedBytes: number;
|
||||||
@@ -111,6 +115,8 @@ export const DEFAULT_ONEPKG_LIMITS: Readonly<OnePkgLimits> = {
|
|||||||
maxNameBytes: 255,
|
maxNameBytes: 255,
|
||||||
maxPathCharacters: 4_096,
|
maxPathCharacters: 4_096,
|
||||||
maxPathDepth: 64,
|
maxPathDepth: 64,
|
||||||
|
maxTocOperations: 1_000_000,
|
||||||
|
maxTocDiagnostics: 10_000,
|
||||||
maxFileBytes: 256 * 1024 * 1024,
|
maxFileBytes: 256 * 1024 * 1024,
|
||||||
maxFolderUncompressedBytes: 512 * 1024 * 1024,
|
maxFolderUncompressedBytes: 512 * 1024 * 1024,
|
||||||
maxTotalUncompressedBytes: 512 * 1024 * 1024,
|
maxTotalUncompressedBytes: 512 * 1024 * 1024,
|
||||||
|
|||||||
Reference in New Issue
Block a user