// This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. // SPDX-License-Identifier: MPL-2.0 // // Portions adapted from onenote.rs (MPL-2.0), revision // 5138a39a3f4e72b840932f9872fecde52fa9da60: `onenote/{section,page_series, // page,outline,rich_text}.rs` and the corresponding `one/property_set/*.rs` // modules. Deviations: title/plain-text traversal only, bounded graph walking, // serializable DTOs, and diagnostics for unsupported content. import { BinaryReader } from '../binary/BinaryReader.js'; import { filetimeToIso, oneNoteTimeToIso } from '../binary/time.js'; import { appendItems } from '../model/append.js'; import type { ParserDiagnostic } from '../model/diagnostics.js'; import type { OneNotePageSummaryDto, OneNoteSectionDto } from '../model/dto.js'; import type { OneNoteParserLimits } from '../parser/limits.js'; import { countObjectReferences, countObjectSpaceReferences, findProperty, referenceOffset, } from '../onestore/property-set.js'; import type { DesktopOneStore } from '../onestore/desktop-store.js'; import type { ExGuid, ObjectSpace, PropertyEntry, StoreObject, StoreObjectSpaceReference, } from '../onestore/types.js'; import { exGuidKey } from '../onestore/types.js'; import { JCID, PROPERTY } from './constants.js'; interface OneContext { store: DesktopOneStore; limits: OneNoteParserLimits; diagnostics: ParserDiagnostic[]; extractedChars: number; currentPage?: string; } type PageWithTextDto = OneNotePageSummaryDto & { text: string }; export function buildSectionDto( store: DesktopOneStore, sourceName: string, limits: OneNoteParserLimits, diagnostics: ParserDiagnostic[] ): OneNoteSectionDto { const context: OneContext = { store, limits, diagnostics, extractedChars: 0, }; const root = store.rootObjectSpace; const metadata = rootObject(root, 2, 'section metadata root'); assertJcid(metadata, JCID.SectionMetadata, 'SectionMetadata'); const displayName = cleanMetadataString( optionalString(metadata, PROPERTY.SectionDisplayName) ); const section = rootObject(root, 1, 'section content root'); assertJcid(section, JCID.SectionNode, 'SectionNode'); const pageSeriesIds = objectReferences(section, PROPERTY.ElementChildNodes); const pages: PageWithTextDto[] = []; for (const seriesId of pageSeriesIds) { const series = getObject(root, seriesId, 'PageSeriesNode'); assertJcid(series, JCID.PageSeriesNode, 'PageSeriesNode'); const pageSpaceIds = objectSpaceReferences( series, PROPERTY.ChildGraphSpaceElementNodes ); for (const pageSpaceReference of pageSpaceIds) { context.currentPage = undefined; const pageSpace = store.objectSpaces.get(pageSpaceReference.key); if (!pageSpace) { addDiagnostic( context, 'MISSING_PAGE_SPACE', `Page object space ${pageSpaceReference.key} is missing`, 'PageSeriesNode' ); continue; } try { if (pages.length >= limits.maxPages) { addDiagnostic( context, 'PAGE_LIMIT_REACHED', `Page count exceeds ${limits.maxPages}`, 'PageSeriesNode', 'error' ); break; } pages.push(parsePage(context, pageSpace)); } catch (error) { const message = error instanceof Error ? error.message : String(error); addDiagnostic( context, 'PAGE_PARSE_FAILED', `Could not parse page ${pageSpaceReference.key}: ${message}`, 'Page', 'error' ); } } if (pages.length >= limits.maxPages) break; } return { format: 'one', sourceName, sectionName: displayName ?? stripOneExtension(sourceName), pages, diagnostics, parserInfo: { implementation: 'typescript', supportedVariant: store.variant, referenceRevision: 'onenote.rs@5138a39a3f4e72b840932f9872fecde52fa9da60', }, }; } function parsePage(context: OneContext, space: ObjectSpace): PageWithTextDto { const metadata = rootObject(space, 2, 'page metadata root'); assertJcid(metadata, JCID.PageMetadata, 'PageMetadata'); const pageGuid = requiredGuid( metadata, PROPERTY.NotebookManagementEntityGuid, 'PageMetadata entity GUID' ); context.currentPage = pageGuid; const created = optionalU64(metadata, PROPERTY.TopologyCreationTimeStamp); const level = optionalU32(metadata, PROPERTY.PageLevel); const cachedMetadataTitle = optionalString( metadata, PROPERTY.CachedTitleString ); const manifest = rootObject(space, 1, 'page content root'); assertJcid(manifest, JCID.PageManifestNode, 'PageManifestNode'); const pageId = objectReferences(manifest, PROPERTY.ContentChildNodes)[0]; if (!pageId) throw new Error('PageManifestNode has no page reference'); const page = getObject(space, pageId, 'PageNode'); assertJcid(page, JCID.PageNode, 'PageNode'); const updated = optionalU32(page, PROPERTY.LastModifiedTime); const cachedPageTitle = optionalString( page, PROPERTY.CachedTitleStringFromPage ); const titleId = objectReferences( page, PROPERTY.StructureElementChildNodes )[0]; const contentIds = objectReferences(page, PROPERTY.ElementChildNodes); let titleText: string | undefined; if (titleId) { try { const title = getObject(space, titleId, 'TitleNode'); assertJcid(title, JCID.TitleNode, 'TitleNode'); const titleOutlines = objectReferences(title, PROPERTY.ElementChildNodes); for (const outlineId of titleOutlines) { const paragraphs = extractGraphTextSafely( context, space, outlineId, 'title' ); titleText = paragraphs.map(cleanTitle).find((item) => item.length > 0); if (titleText) break; } } catch (error) { addDiagnostic( context, 'TITLE_PARSE_FAILED', error instanceof Error ? error.message : String(error), 'TitleNode' ); } } const bodyParagraphs: string[] = []; for (const contentId of contentIds) { bodyParagraphs.push( ...extractGraphTextSafely(context, space, contentId, 'page content') ); } let text = bodyParagraphs .map((paragraph) => paragraph.split('\u000b').join('\n').trimEnd()) .filter((paragraph) => paragraph.length > 0) .join('\n'); const remaining = context.limits.maxExtractedTextChars - context.extractedChars; if (text.length > remaining) { text = text.slice(0, Math.max(0, remaining)); addDiagnostic( context, 'TEXT_LIMIT_REACHED', `Extracted text was truncated at ${context.limits.maxExtractedTextChars} characters`, `Page ${pageGuid}` ); } context.extractedChars += text.length; const bodyFallback = bodyParagraphs .map(cleanTitle) .find((item) => item.length > 0); const title = titleText ?? bodyFallback ?? cachedPageTitle ?? cachedMetadataTitle ?? 'Untitled page'; return { id: pageGuid, title, createdAt: created === undefined ? undefined : safeFiletime(context, created), updatedAt: updated === undefined ? undefined : oneNoteTimeToIso(updated), level, text, textPreview: preview(text), }; } function extractGraphTextSafely( context: OneContext, space: ObjectSpace, id: ExGuid, label: string ): string[] { try { return extractGraphText(context, space, id, new Set(), 0); } catch (error) { addDiagnostic( context, 'CONTENT_PARSE_FAILED', `Could not extract ${label} ${exGuidKey(id)}: ${error instanceof Error ? error.message : String(error)}`, 'MS-ONE content graph' ); return []; } } function extractGraphText( context: OneContext, space: ObjectSpace, id: ExGuid, active: Set, depth: number ): string[] { if (depth > context.limits.maxGraphDepth) { addDiagnostic( context, 'GRAPH_DEPTH_LIMIT', `Content graph depth exceeds ${context.limits.maxGraphDepth}`, 'MS-ONE content graph' ); return []; } const key = exGuidKey(id); if (active.has(key)) { addDiagnostic( context, 'CONTENT_GRAPH_CYCLE', `Cycle detected at object ${key}`, 'MS-ONE content graph' ); return []; } const object = space.objects.get(key); if (!object) { addDiagnostic( context, 'MISSING_CONTENT_OBJECT', `Content object ${key} is missing`, 'MS-ONE content graph' ); return []; } active.add(key); try { if (object.jcid === JCID.RichTextNode) { const text = richText(context, space, object); return text === undefined ? [] : [text]; } if (object.jcid === JCID.OutlineNode || object.jcid === JCID.OutlineGroup) { return traverseReferences( context, space, objectReferences(object, PROPERTY.ElementChildNodes), active, depth ); } if (object.jcid === JCID.OutlineElementNode) { const contents = traverseReferences( context, space, objectReferences(object, PROPERTY.ContentChildNodes), active, depth ); const children = traverseReferences( context, space, objectReferences(object, PROPERTY.ElementChildNodes), active, depth ); return [...contents, ...children]; } addDiagnostic( context, unsupportedCode(object.jcid), `Content object JCID 0x${object.jcid.toString(16)} is not yet rendered`, `Object ${key}` ); return []; } finally { active.delete(key); } } function traverseReferences( context: OneContext, space: ObjectSpace, ids: ExGuid[], active: Set, depth: number ): string[] { const result: string[] = []; for (const id of ids) { appendItems( result, extractGraphText(context, space, id, active, depth + 1) ); } return result; } function richText( context: OneContext, space: ObjectSpace, object: StoreObject ): string | undefined { let text = optionalString(object, PROPERTY.RichEditTextUnicode); if (text === undefined) text = optionalLatin1(object, PROPERTY.TextExtendedAscii); if (text === undefined) return undefined; const styles = objectReferences(object, PROPERTY.TextRunFormatting); if (styles.length === 0) return text.split('\0').join(''); const indices = optionalU32Vector(object, PROPERTY.TextRunIndex) ?? []; const hidden = styles.map((styleId) => { const style = space.objects.get(exGuidKey(styleId)); if (!style) { addDiagnostic( context, 'MISSING_TEXT_STYLE', `Text-run style ${exGuidKey(styleId)} is missing`, 'RichTextNode' ); return false; } return optionalBool(style, PROPERTY.Hidden) ?? false; }); const boundaries = [0, ...indices]; if (boundaries.length === hidden.length) boundaries.push(text.length); let visible = ''; for (let index = 0; index < hidden.length; index += 1) { if (hidden[index]) continue; const start = Math.min(boundaries[index] ?? text.length, text.length); const end = Math.min(boundaries[index + 1] ?? text.length, text.length); if (start < end) visible += text.slice(start, end); } // OneNote producers sometimes retain UTF-16 terminators inside text vectors. return visible.split('\0').join(''); } function objectReferences(object: StoreObject, propertyId: number): ExGuid[] { const property = findProperty(object.props, propertyId); if (!property) return []; let count: number; if (property.value.kind === 'objectId') count = 1; else if (property.value.kind === 'objectIds') count = property.value.count; else throw new Error( `Property 0x${propertyId.toString(16)} is not an object reference` ); const offset = referenceOffset( object.props, propertyId, countObjectReferences ); if (object.resolvedObjectIds) { return resolvedReferenceSlice(object.resolvedObjectIds, offset, count); } return resolveReferenceSlice(object, object.props.objectIds, offset, count); } function objectSpaceReferences( object: StoreObject, propertyId: number ): StoreObjectSpaceReference[] { const property = findProperty(object.props, propertyId); if (!property) return []; let count: number; if (property.value.kind === 'objectSpaceId') count = 1; else if (property.value.kind === 'objectSpaceIds') count = property.value.count; else { throw new Error( `Property 0x${propertyId.toString(16)} is not an object-space reference` ); } const offset = referenceOffset( object.props, propertyId, countObjectSpaceReferences ); if (object.resolvedObjectSpaceIds) { return resolvedReferenceSlice(object.resolvedObjectSpaceIds, offset, count); } return resolveReferenceSlice( object, object.props.objectSpaceIds, offset, count ).map((id) => ({ id, key: exGuidKey(id) })); } function resolveReferenceSlice( object: StoreObject, ids: { guidIndex: number; value: number }[], offset: number, count: number ): ExGuid[] { if (offset + count > ids.length) { throw new Error( `Reference range ${offset}+${count} exceeds stream length ${ids.length}` ); } return ids.slice(offset, offset + count).map((id) => { const guid = object.mapping.get(id.guidIndex); if (!guid) throw new Error(`GUID mapping ${id.guidIndex} is missing`); return { guid, value: id.value }; }); } function resolvedReferenceSlice( ids: readonly T[], offset: number, count: number ): T[] { if (offset + count > ids.length) { throw new Error( `Reference range ${offset}+${count} exceeds resolved stream length ${ids.length}` ); } return ids.slice(offset, offset + count); } function rootObject( space: ObjectSpace, role: number, structure: string ): StoreObject { const id = space.roots.get(role); if (!id) throw new Error(`Object space has no ${structure}`); return getObject(space, id, structure); } function getObject( space: ObjectSpace, id: ExGuid, structure: string ): StoreObject { const object = space.objects.get(exGuidKey(id)); if (!object) throw new Error(`${structure} object ${exGuidKey(id)} is missing`); return object; } function assertJcid( object: StoreObject, expected: number, structure: string ): void { if (object.jcid !== expected) { throw new Error( `${structure} has JCID 0x${object.jcid.toString(16)}, expected 0x${expected.toString(16)}` ); } } function optionalEntry( object: StoreObject, propertyId: number ): PropertyEntry | undefined { return findProperty(object.props, propertyId); } function optionalString( object: StoreObject, propertyId: number ): string | undefined { const entry = optionalEntry(object, propertyId); if (!entry) return undefined; if (entry.value.kind !== 'bytes') { throw new Error( `Property 0x${propertyId.toString(16)} is not a byte vector` ); } if (entry.value.value.byteLength % 2 !== 0) { throw new Error( `UTF-16 property 0x${propertyId.toString(16)} has odd length` ); } return new TextDecoder('utf-16le', { fatal: true }).decode(entry.value.value); } function optionalLatin1( object: StoreObject, propertyId: number ): string | undefined { const entry = optionalEntry(object, propertyId); if (!entry) return undefined; if (entry.value.kind !== 'bytes') { throw new Error( `Property 0x${propertyId.toString(16)} is not a byte vector` ); } let result = ''; for (const value of entry.value.value) result += String.fromCharCode(value); return result; } function requiredGuid( object: StoreObject, propertyId: number, structure: string ): string { const entry = optionalEntry(object, propertyId); if ( !entry || entry.value.kind !== 'bytes' || entry.value.value.length !== 16 ) { throw new Error(`${structure} is absent or not 16 bytes`); } const reader = new BinaryReader(entry.value.value, 0, 16, structure); const bytes = reader.read(16); const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15]; const hex = order.map((index) => bytes[index]!.toString(16).padStart(2, '0')); return `{${hex.slice(0, 4).join('')}-${hex.slice(4, 6).join('')}-${hex.slice(6, 8).join('')}-${hex.slice(8, 10).join('')}-${hex.slice(10).join('')}}`.toUpperCase(); } function optionalBool( object: StoreObject, propertyId: number ): boolean | undefined { const value = optionalEntry(object, propertyId)?.value; if (!value) return undefined; if (value.kind !== 'bool') throw new Error('Expected a Boolean property'); return value.value; } function optionalU32( object: StoreObject, propertyId: number ): number | undefined { const value = optionalEntry(object, propertyId)?.value; if (!value) return undefined; if (value.kind !== 'u32') throw new Error('Expected a u32 property'); return value.value; } function optionalU64( object: StoreObject, propertyId: number ): bigint | undefined { const value = optionalEntry(object, propertyId)?.value; if (!value) return undefined; if (value.kind !== 'u64') throw new Error('Expected a u64 property'); return value.value; } function optionalU32Vector( object: StoreObject, propertyId: number ): number[] | undefined { const value = optionalEntry(object, propertyId)?.value; if (!value) return undefined; if (value.kind !== 'bytes' || value.value.byteLength % 4 !== 0) { throw new Error('Expected a u32 vector property'); } const reader = new BinaryReader( value.value, 0, value.value.byteLength, 'u32 vector' ); const result: number[] = []; while (reader.remaining > 0) result.push(reader.u32()); return result; } function safeFiletime(context: OneContext, value: bigint): string | undefined { try { return filetimeToIso(value, 'PageMetadata creation time'); } catch (error) { addDiagnostic( context, 'INVALID_PAGE_TIMESTAMP', error instanceof Error ? error.message : String(error), 'PageMetadata' ); return undefined; } } function cleanTitle(text: string): string { const marker = '\ufddfHYPERLINK "'; let result = text; while (result.startsWith('\u000b')) result = result.slice(1); result = result.trim(); let start = result.indexOf(marker); while (start >= 0) { const valueStart = start + marker.length; const quote = result.indexOf('"', valueStart); result = quote < 0 ? result.slice(0, start) : result.slice(0, start) + result.slice(quote + 1); start = result.indexOf(marker); } return result.trim(); } function preview(text: string): string { const compact = text.replace(/\s+/g, ' ').trim(); return compact.length <= 500 ? compact : `${compact.slice(0, 497)}…`; } function unsupportedCode(jcid: number): string { if (jcid === JCID.ImageNode) return 'UNSUPPORTED_IMAGE'; if (jcid === JCID.TableNode) return 'UNSUPPORTED_TABLE'; if (jcid === JCID.EmbeddedFileNode) return 'UNSUPPORTED_EMBEDDED_FILE'; if (jcid === JCID.InkContainer) return 'UNSUPPORTED_INK'; return 'UNSUPPORTED_CONTENT_OBJECT'; } function addDiagnostic( context: OneContext, code: string, message: string, structure: string, severity: 'warning' | 'error' = 'warning' ): void { if (context.diagnostics.length >= context.limits.maxDiagnostics) return; context.diagnostics.push({ severity, code, message, structure: context.currentPage ? `${structure} (page ${context.currentPage})` : structure, recoverable: true, }); } function stripOneExtension(filename: string): string { return (filename.split(/[\\/]/).at(-1) ?? filename).replace(/\.one$/i, ''); } function cleanMetadataString(value: string | undefined): string | undefined { const result = value?.split('\0').join('').trim(); return result || undefined; }