feat: add local OneNote and ONEPKG reader

This commit is contained in:
2026-07-22 17:06:03 +02:00
commit f581cbdced
93 changed files with 14949 additions and 0 deletions

661
src/onenote/one/section.ts Normal file
View File

@@ -0,0 +1,661 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
//
// Portions adapted from onenote.rs (MPL-2.0), revision
// 5138a39a3f4e72b840932f9872fecde52fa9da60: `onenote/{section,page_series,
// page,outline,rich_text}.rs` and the corresponding `one/property_set/*.rs`
// modules. Deviations: title/plain-text traversal only, bounded graph walking,
// serializable DTOs, and diagnostics for unsupported content.
import { BinaryReader } from '../binary/BinaryReader.js';
import { filetimeToIso, oneNoteTimeToIso } from '../binary/time.js';
import type { ParserDiagnostic } from '../model/diagnostics.js';
import type { OneNotePageSummaryDto, OneNoteSectionDto } from '../model/dto.js';
import type { OneNoteParserLimits } from '../parser/limits.js';
import {
countObjectReferences,
countObjectSpaceReferences,
findProperty,
referenceOffset,
} from '../onestore/property-set.js';
import type { DesktopOneStore } from '../onestore/desktop-store.js';
import type {
ExGuid,
ObjectSpace,
PropertyEntry,
StoreObject,
} from '../onestore/types.js';
import { exGuidKey } from '../onestore/types.js';
import { JCID, PROPERTY } from './constants.js';
interface OneContext {
store: DesktopOneStore;
limits: OneNoteParserLimits;
diagnostics: ParserDiagnostic[];
extractedChars: number;
currentPage?: string;
}
type PageWithTextDto = OneNotePageSummaryDto & { text: string };
export function buildSectionDto(
store: DesktopOneStore,
sourceName: string,
limits: OneNoteParserLimits,
diagnostics: ParserDiagnostic[]
): OneNoteSectionDto {
const context: OneContext = {
store,
limits,
diagnostics,
extractedChars: 0,
};
const root = store.rootObjectSpace;
const metadata = rootObject(root, 2, 'section metadata root');
assertJcid(metadata, JCID.SectionMetadata, 'SectionMetadata');
const displayName = optionalString(metadata, PROPERTY.SectionDisplayName);
const section = rootObject(root, 1, 'section content root');
assertJcid(section, JCID.SectionNode, 'SectionNode');
const pageSeriesIds = objectReferences(section, PROPERTY.ElementChildNodes);
const pages: PageWithTextDto[] = [];
for (const seriesId of pageSeriesIds) {
const series = getObject(root, seriesId, 'PageSeriesNode');
assertJcid(series, JCID.PageSeriesNode, 'PageSeriesNode');
const pageSpaceIds = objectSpaceReferences(
series,
PROPERTY.ChildGraphSpaceElementNodes
);
for (const pageSpaceId of pageSpaceIds) {
context.currentPage = undefined;
const pageSpace = store.objectSpaces.get(exGuidKey(pageSpaceId));
if (!pageSpace) {
addDiagnostic(
context,
'MISSING_PAGE_SPACE',
`Page object space ${exGuidKey(pageSpaceId)} is missing`,
'PageSeriesNode'
);
continue;
}
try {
if (pages.length >= limits.maxPages) {
addDiagnostic(
context,
'PAGE_LIMIT_REACHED',
`Page count exceeds ${limits.maxPages}`,
'PageSeriesNode',
'error'
);
break;
}
pages.push(parsePage(context, pageSpace));
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
addDiagnostic(
context,
'PAGE_PARSE_FAILED',
`Could not parse page ${exGuidKey(pageSpaceId)}: ${message}`,
'Page',
'error'
);
}
}
if (pages.length >= limits.maxPages) break;
}
return {
format: 'one',
sourceName,
sectionName: displayName ?? stripOneExtension(sourceName),
pages,
diagnostics,
parserInfo: {
implementation: 'typescript',
supportedVariant: 'desktop-revision-store',
referenceRevision: 'onenote.rs@5138a39a3f4e72b840932f9872fecde52fa9da60',
},
};
}
function parsePage(context: OneContext, space: ObjectSpace): PageWithTextDto {
const metadata = rootObject(space, 2, 'page metadata root');
assertJcid(metadata, JCID.PageMetadata, 'PageMetadata');
const pageGuid = requiredGuid(
metadata,
PROPERTY.NotebookManagementEntityGuid,
'PageMetadata entity GUID'
);
context.currentPage = pageGuid;
const created = optionalU64(metadata, PROPERTY.TopologyCreationTimeStamp);
const level = optionalU32(metadata, PROPERTY.PageLevel);
const cachedMetadataTitle = optionalString(
metadata,
PROPERTY.CachedTitleString
);
const manifest = rootObject(space, 1, 'page content root');
assertJcid(manifest, JCID.PageManifestNode, 'PageManifestNode');
const pageId = objectReferences(manifest, PROPERTY.ContentChildNodes)[0];
if (!pageId) throw new Error('PageManifestNode has no page reference');
const page = getObject(space, pageId, 'PageNode');
assertJcid(page, JCID.PageNode, 'PageNode');
const updated = optionalU32(page, PROPERTY.LastModifiedTime);
const cachedPageTitle = optionalString(
page,
PROPERTY.CachedTitleStringFromPage
);
const titleId = objectReferences(
page,
PROPERTY.StructureElementChildNodes
)[0];
const contentIds = objectReferences(page, PROPERTY.ElementChildNodes);
let titleText: string | undefined;
if (titleId) {
try {
const title = getObject(space, titleId, 'TitleNode');
assertJcid(title, JCID.TitleNode, 'TitleNode');
const titleOutlines = objectReferences(title, PROPERTY.ElementChildNodes);
for (const outlineId of titleOutlines) {
const paragraphs = extractGraphTextSafely(
context,
space,
outlineId,
'title'
);
titleText = paragraphs.map(cleanTitle).find((item) => item.length > 0);
if (titleText) break;
}
} catch (error) {
addDiagnostic(
context,
'TITLE_PARSE_FAILED',
error instanceof Error ? error.message : String(error),
'TitleNode'
);
}
}
const bodyParagraphs: string[] = [];
for (const contentId of contentIds) {
bodyParagraphs.push(
...extractGraphTextSafely(context, space, contentId, 'page content')
);
}
let text = bodyParagraphs
.map((paragraph) => paragraph.split('\u000b').join('\n').trimEnd())
.filter((paragraph) => paragraph.length > 0)
.join('\n');
const remaining =
context.limits.maxExtractedTextChars - context.extractedChars;
if (text.length > remaining) {
text = text.slice(0, Math.max(0, remaining));
addDiagnostic(
context,
'TEXT_LIMIT_REACHED',
`Extracted text was truncated at ${context.limits.maxExtractedTextChars} characters`,
`Page ${pageGuid}`
);
}
context.extractedChars += text.length;
const bodyFallback = bodyParagraphs
.map(cleanTitle)
.find((item) => item.length > 0);
const title =
titleText ??
bodyFallback ??
cachedPageTitle ??
cachedMetadataTitle ??
'Untitled page';
return {
id: pageGuid,
title,
createdAt:
created === undefined ? undefined : safeFiletime(context, created),
updatedAt: updated === undefined ? undefined : oneNoteTimeToIso(updated),
level,
text,
textPreview: preview(text),
};
}
function extractGraphTextSafely(
context: OneContext,
space: ObjectSpace,
id: ExGuid,
label: string
): string[] {
try {
return extractGraphText(context, space, id, new Set(), 0);
} catch (error) {
addDiagnostic(
context,
'CONTENT_PARSE_FAILED',
`Could not extract ${label} ${exGuidKey(id)}: ${error instanceof Error ? error.message : String(error)}`,
'MS-ONE content graph'
);
return [];
}
}
function extractGraphText(
context: OneContext,
space: ObjectSpace,
id: ExGuid,
active: Set<string>,
depth: number
): string[] {
if (depth > context.limits.maxGraphDepth) {
addDiagnostic(
context,
'GRAPH_DEPTH_LIMIT',
`Content graph depth exceeds ${context.limits.maxGraphDepth}`,
'MS-ONE content graph'
);
return [];
}
const key = exGuidKey(id);
if (active.has(key)) {
addDiagnostic(
context,
'CONTENT_GRAPH_CYCLE',
`Cycle detected at object ${key}`,
'MS-ONE content graph'
);
return [];
}
const object = space.objects.get(key);
if (!object) {
addDiagnostic(
context,
'MISSING_CONTENT_OBJECT',
`Content object ${key} is missing`,
'MS-ONE content graph'
);
return [];
}
active.add(key);
try {
if (object.jcid === JCID.RichTextNode) {
const text = richText(context, space, object);
return text === undefined ? [] : [text];
}
if (object.jcid === JCID.OutlineNode || object.jcid === JCID.OutlineGroup) {
return traverseReferences(
context,
space,
objectReferences(object, PROPERTY.ElementChildNodes),
active,
depth
);
}
if (object.jcid === JCID.OutlineElementNode) {
const contents = traverseReferences(
context,
space,
objectReferences(object, PROPERTY.ContentChildNodes),
active,
depth
);
const children = traverseReferences(
context,
space,
objectReferences(object, PROPERTY.ElementChildNodes),
active,
depth
);
return [...contents, ...children];
}
addDiagnostic(
context,
unsupportedCode(object.jcid),
`Content object JCID 0x${object.jcid.toString(16)} is not yet rendered`,
`Object ${key}`
);
return [];
} finally {
active.delete(key);
}
}
function traverseReferences(
context: OneContext,
space: ObjectSpace,
ids: ExGuid[],
active: Set<string>,
depth: number
): string[] {
const result: string[] = [];
for (const id of ids) {
result.push(...extractGraphText(context, space, id, active, depth + 1));
}
return result;
}
function richText(
context: OneContext,
space: ObjectSpace,
object: StoreObject
): string | undefined {
let text = optionalString(object, PROPERTY.RichEditTextUnicode);
if (text === undefined)
text = optionalLatin1(object, PROPERTY.TextExtendedAscii);
if (text === undefined) return undefined;
const styles = objectReferences(object, PROPERTY.TextRunFormatting);
if (styles.length === 0) return text.split('\0').join('');
const indices = optionalU32Vector(object, PROPERTY.TextRunIndex) ?? [];
const hidden = styles.map((styleId) => {
const style = space.objects.get(exGuidKey(styleId));
if (!style) {
addDiagnostic(
context,
'MISSING_TEXT_STYLE',
`Text-run style ${exGuidKey(styleId)} is missing`,
'RichTextNode'
);
return false;
}
return optionalBool(style, PROPERTY.Hidden) ?? false;
});
const boundaries = [0, ...indices];
if (boundaries.length === hidden.length) boundaries.push(text.length);
let visible = '';
for (let index = 0; index < hidden.length; index += 1) {
if (hidden[index]) continue;
const start = Math.min(boundaries[index] ?? text.length, text.length);
const end = Math.min(boundaries[index + 1] ?? text.length, text.length);
if (start < end) visible += text.slice(start, end);
}
// OneNote producers sometimes retain UTF-16 terminators inside text vectors.
return visible.split('\0').join('');
}
function objectReferences(object: StoreObject, propertyId: number): ExGuid[] {
const property = findProperty(object.props, propertyId);
if (!property) return [];
let count: number;
if (property.value.kind === 'objectId') count = 1;
else if (property.value.kind === 'objectIds') count = property.value.count;
else
throw new Error(
`Property 0x${propertyId.toString(16)} is not an object reference`
);
const offset = referenceOffset(
object.props,
propertyId,
countObjectReferences
);
return resolveReferenceSlice(object, object.props.objectIds, offset, count);
}
function objectSpaceReferences(
object: StoreObject,
propertyId: number
): ExGuid[] {
const property = findProperty(object.props, propertyId);
if (!property) return [];
let count: number;
if (property.value.kind === 'objectSpaceId') count = 1;
else if (property.value.kind === 'objectSpaceIds')
count = property.value.count;
else {
throw new Error(
`Property 0x${propertyId.toString(16)} is not an object-space reference`
);
}
const offset = referenceOffset(
object.props,
propertyId,
countObjectSpaceReferences
);
return resolveReferenceSlice(
object,
object.props.objectSpaceIds,
offset,
count
);
}
function resolveReferenceSlice(
object: StoreObject,
ids: { guidIndex: number; value: number }[],
offset: number,
count: number
): ExGuid[] {
if (offset + count > ids.length) {
throw new Error(
`Reference range ${offset}+${count} exceeds stream length ${ids.length}`
);
}
return ids.slice(offset, offset + count).map((id) => {
const guid = object.mapping.get(id.guidIndex);
if (!guid) throw new Error(`GUID mapping ${id.guidIndex} is missing`);
return { guid, value: id.value };
});
}
function rootObject(
space: ObjectSpace,
role: number,
structure: string
): StoreObject {
const id = space.roots.get(role);
if (!id) throw new Error(`Object space has no ${structure}`);
return getObject(space, id, structure);
}
function getObject(
space: ObjectSpace,
id: ExGuid,
structure: string
): StoreObject {
const object = space.objects.get(exGuidKey(id));
if (!object)
throw new Error(`${structure} object ${exGuidKey(id)} is missing`);
return object;
}
function assertJcid(
object: StoreObject,
expected: number,
structure: string
): void {
if (object.jcid !== expected) {
throw new Error(
`${structure} has JCID 0x${object.jcid.toString(16)}, expected 0x${expected.toString(16)}`
);
}
}
function optionalEntry(
object: StoreObject,
propertyId: number
): PropertyEntry | undefined {
return findProperty(object.props, propertyId);
}
function optionalString(
object: StoreObject,
propertyId: number
): string | undefined {
const entry = optionalEntry(object, propertyId);
if (!entry) return undefined;
if (entry.value.kind !== 'bytes') {
throw new Error(
`Property 0x${propertyId.toString(16)} is not a byte vector`
);
}
if (entry.value.value.byteLength % 2 !== 0) {
throw new Error(
`UTF-16 property 0x${propertyId.toString(16)} has odd length`
);
}
return new TextDecoder('utf-16le', { fatal: true }).decode(entry.value.value);
}
function optionalLatin1(
object: StoreObject,
propertyId: number
): string | undefined {
const entry = optionalEntry(object, propertyId);
if (!entry) return undefined;
if (entry.value.kind !== 'bytes') {
throw new Error(
`Property 0x${propertyId.toString(16)} is not a byte vector`
);
}
let result = '';
for (const value of entry.value.value) result += String.fromCharCode(value);
return result;
}
function requiredGuid(
object: StoreObject,
propertyId: number,
structure: string
): string {
const entry = optionalEntry(object, propertyId);
if (
!entry ||
entry.value.kind !== 'bytes' ||
entry.value.value.length !== 16
) {
throw new Error(`${structure} is absent or not 16 bytes`);
}
const reader = new BinaryReader(entry.value.value, 0, 16, structure);
const bytes = reader.read(16);
const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15];
const hex = order.map((index) => bytes[index]!.toString(16).padStart(2, '0'));
return `{${hex.slice(0, 4).join('')}-${hex.slice(4, 6).join('')}-${hex.slice(6, 8).join('')}-${hex.slice(8, 10).join('')}-${hex.slice(10).join('')}}`.toUpperCase();
}
function optionalBool(
object: StoreObject,
propertyId: number
): boolean | undefined {
const value = optionalEntry(object, propertyId)?.value;
if (!value) return undefined;
if (value.kind !== 'bool') throw new Error('Expected a Boolean property');
return value.value;
}
function optionalU32(
object: StoreObject,
propertyId: number
): number | undefined {
const value = optionalEntry(object, propertyId)?.value;
if (!value) return undefined;
if (value.kind !== 'u32') throw new Error('Expected a u32 property');
return value.value;
}
function optionalU64(
object: StoreObject,
propertyId: number
): bigint | undefined {
const value = optionalEntry(object, propertyId)?.value;
if (!value) return undefined;
if (value.kind !== 'u64') throw new Error('Expected a u64 property');
return value.value;
}
function optionalU32Vector(
object: StoreObject,
propertyId: number
): number[] | undefined {
const value = optionalEntry(object, propertyId)?.value;
if (!value) return undefined;
if (value.kind !== 'bytes' || value.value.byteLength % 4 !== 0) {
throw new Error('Expected a u32 vector property');
}
const reader = new BinaryReader(
value.value,
0,
value.value.byteLength,
'u32 vector'
);
const result: number[] = [];
while (reader.remaining > 0) result.push(reader.u32());
return result;
}
function safeFiletime(context: OneContext, value: bigint): string | undefined {
try {
return filetimeToIso(value, 'PageMetadata creation time');
} catch (error) {
addDiagnostic(
context,
'INVALID_PAGE_TIMESTAMP',
error instanceof Error ? error.message : String(error),
'PageMetadata'
);
return undefined;
}
}
function cleanTitle(text: string): string {
const marker = '\ufddfHYPERLINK "';
let result = text;
while (result.startsWith('\u000b')) result = result.slice(1);
result = result.trim();
let start = result.indexOf(marker);
while (start >= 0) {
const valueStart = start + marker.length;
const quote = result.indexOf('"', valueStart);
result =
quote < 0
? result.slice(0, start)
: result.slice(0, start) + result.slice(quote + 1);
start = result.indexOf(marker);
}
return result.trim();
}
function preview(text: string): string {
const compact = text.replace(/\s+/g, ' ').trim();
return compact.length <= 500 ? compact : `${compact.slice(0, 497)}`;
}
function unsupportedCode(jcid: number): string {
if (jcid === JCID.ImageNode) return 'UNSUPPORTED_IMAGE';
if (jcid === JCID.TableNode) return 'UNSUPPORTED_TABLE';
if (jcid === JCID.EmbeddedFileNode) return 'UNSUPPORTED_EMBEDDED_FILE';
if (jcid === JCID.InkContainer) return 'UNSUPPORTED_INK';
return 'UNSUPPORTED_CONTENT_OBJECT';
}
function addDiagnostic(
context: OneContext,
code: string,
message: string,
structure: string,
severity: 'warning' | 'error' = 'warning'
): void {
if (context.diagnostics.length >= context.limits.maxDiagnostics) return;
context.diagnostics.push({
severity,
code,
message,
structure: context.currentPage
? `${structure} (page ${context.currentPage})`
: structure,
recoverable: true,
});
}
function stripOneExtension(filename: string): string {
return (filename.split(/[\\/]/).at(-1) ?? filename).replace(/\.one$/i, '');
}