diff --git a/src/onenote/index.ts b/src/onenote/index.ts index c16dc81..cb691a1 100644 --- a/src/onenote/index.ts +++ b/src/onenote/index.ts @@ -10,6 +10,20 @@ export { } from './parser/limits.js'; export { parseOneNoteSection, + parseOneNoteSectionContent, type ParseOneNoteSectionOptions, } from './parser/parse-section.js'; export type { OneNotePageSummaryDto, OneNoteSectionDto } from './model/dto.js'; +export type { + OneNoteAttachmentBlock, + OneNoteColor, + OneNoteContentBlock, + OneNoteImageBlock, + OneNotePageContentModel, + OneNoteResource, + OneNoteSectionContentModel, + OneNoteTableBlock, + OneNoteTextBlock, + OneNoteTextRun, + OneNoteTextStyle, +} from './one/content-model.js'; diff --git a/src/onenote/one/constants.ts b/src/onenote/one/constants.ts index bbc813b..acd2847 100644 --- a/src/onenote/one/constants.ts +++ b/src/onenote/one/constants.ts @@ -15,11 +15,20 @@ export const JCID = { OutlineElementNode: 0x0006000d, RichTextNode: 0x0006000e, ImageNode: 0x00060011, + NumberListNode: 0x00060012, InkContainer: 0x00060014, OutlineGroup: 0x00060019, TableNode: 0x00060022, + TableRowNode: 0x00060023, + TableCellNode: 0x00060024, TitleNode: 0x0006002c, EmbeddedFileNode: 0x00060035, + EmbeddedFileContainer: 0x00080036, + PictureContainer: 0x00080039, + XpsContainer: 0x0008003a, + InkDataNode: 0x0002003b, + InkStrokeNode: 0x00020047, + StrokePropertiesNode: 0x00120048, PageManifestNode: 0x00060037, PageMetadata: 0x00020030, SectionMetadata: 0x00020031, @@ -43,5 +52,71 @@ export const PROPERTY = { TextExtendedAscii: 0x1c003498, TextRunFormatting: 0x24001e13, TextRunIndex: 0x1c001e12, + ParagraphStyle: 0x2000342c, + ParagraphAlignment: 0x0c003477, + ParagraphSpaceBefore: 0x1400342e, + ParagraphSpaceAfter: 0x1400342f, + ParagraphLineSpacingExact: 0x14003430, + ReadingOrderRtl: 0x08003476, + Bold: 0x08001c04, + Italic: 0x08001c05, + Underline: 0x08001c06, + Strikethrough: 0x08001c07, + Superscript: 0x08001c08, + Subscript: 0x08001c09, + Font: 0x1c001c0a, + FontSize: 0x10001c0b, + FontColor: 0x14001c0c, + Highlight: 0x14001c0d, + LanguageId: 0x14001c3b, + RichEditTextLangId: 0x10001cfe, + Charset: 0x0c001d01, + MathFormatting: 0x08003401, + NextStyle: 0x1c00348a, + ParagraphStyleId: 0x1c00345a, + Hyperlink: 0x08001e14, + HyperlinkProtected: 0x08001e19, Hidden: 0x08001e16, + LayoutTightLayout: 0x08001c00, + LayoutAlignmentInParent: 0x14001c3e, + LayoutAlignmentSelf: 0x14001c84, + LayoutMaxWidth: 0x14001c1b, + LayoutMaxHeight: 0x14001c1c, + IsLayoutSizeSetByUser: 0x08001cbd, + OffsetFromParentHoriz: 0x14001c14, + OffsetFromParentVert: 0x14001c15, + PictureContainer: 0x20001c3f, + WebPictureContainer14: 0x200034c8, + ImageAltText: 0x1c001e58, + ImageFilename: 0x1c001dd7, + DisplayedPageNumber: 0x14003480, + PictureWidth: 0x140034cd, + PictureHeight: 0x140034ce, + WzHyperlinkUrl: 0x1c001e20, + IsBackground: 0x08001d13, + EmbeddedFileContainer: 0x20001d9b, + EmbeddedFileName: 0x1c001d9c, + SourceFilepath: 0x1c001d9d, + IRecordMedia: 0x14001d24, + RowCount: 0x14001d57, + ColumnCount: 0x14001d58, + TableBordersVisible: 0x08001d5e, + TableColumnWidths: 0x1c001d66, + TableColumnsLocked: 0x1c001d7d, + CellBackgroundColor: 0x14001e26, + RgOutlineIndentDistance: 0x1c001c12, + OutlineElementChildLevel: 0x0c001c03, + InkData: 0x20003415, + InkStrokes: 0x24003416, + InkScalingX: 0x14001c46, + InkScalingY: 0x14001c47, + InkBoundingBox: 0x1c003418, + InkPath: 0x1c00340b, + InkStrokeProperties: 0x20003409, + InkDimensions: 0x1c00340a, + InkPenTip: 0x0c003412, + InkTransparency: 0x0c003414, + InkHeight: 0x1400340c, + InkWidth: 0x1400340d, + InkColor: 0x1400340f, } as const; diff --git a/src/onenote/one/content-model.ts b/src/onenote/one/content-model.ts new file mode 100644 index 0000000..175c148 --- /dev/null +++ b/src/onenote/one/content-model.ts @@ -0,0 +1,204 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 + +import type { ParserDiagnostic } from '../model/diagnostics.js'; +import type { FileDataBlobReference } from '../onestore/types.js'; + +export interface OneNoteColor { + kind: 'auto' | 'rgb' | 'rgba'; + red?: number; + green?: number; + blue?: number; + alpha?: number; +} + +/** Values are retained in the half-inch unit used by MS-ONE. */ +export interface OneNoteLayout { + maxWidth?: number; + maxHeight?: number; + offsetHorizontal?: number; + offsetVertical?: number; + alignmentInParent?: number; + alignmentSelf?: number; + sizeSetByUser?: boolean; +} + +export interface OneNoteTextStyle { + charset?: number; + bold?: boolean; + italic?: boolean; + underline?: boolean; + strikethrough?: boolean; + superscript?: boolean; + subscript?: boolean; + font?: string; + /** Font size in half-points. */ + fontSize?: number; + fontColor?: OneNoteColor; + highlight?: OneNoteColor; + languageCode?: number; + mathFormatting?: boolean; + hyperlink?: boolean; + hyperlinkProtected?: boolean; + hidden?: boolean; +} + +export interface OneNoteTextRun { + /** UTF-16 offsets into sourceText, matching MS-ONE character positions. */ + start: number; + end: number; + text: string; + style: OneNoteTextStyle; + /** Present only for allow-listed URI schemes. */ + href?: string; +} + +export interface OneNoteTextBlock { + kind: 'text'; + id: string; + sourceText: string; + /** Visible text with hidden hyperlink marker runs and NULs removed. */ + text: string; + runs: OneNoteTextRun[]; + paragraphStyle: OneNoteTextStyle; + paragraphAlignment?: 'left' | 'center' | 'right' | 'unknown'; + paragraphSpaceBefore?: number; + paragraphSpaceAfter?: number; + paragraphLineSpacingExact?: number; + rightToLeft?: boolean; + layout: OneNoteLayout; +} + +export interface OneNoteImageBlock { + kind: 'image'; + id: string; + resourceId?: string; + filename?: string; + altText?: string; + ocrText?: string; + pictureWidth?: number; + pictureHeight?: number; + displayedPageNumber?: number; + href?: string; + isBackground: boolean; + layout: OneNoteLayout; +} + +export interface OneNoteAttachmentBlock { + kind: 'attachment'; + id: string; + resourceId?: string; + iconResourceId?: string; + /** Untrusted display metadata. It must not be used as a filesystem path. */ + filename: string; + size?: number; + layout: OneNoteLayout; +} + +export interface OneNoteTableCell { + id: string; + blocks: OneNoteContentBlock[]; + backgroundColor?: OneNoteColor; + maxWidth?: number; +} + +export interface OneNoteTableRow { + id: string; + cells: OneNoteTableCell[]; +} + +export interface OneNoteTableBlock { + kind: 'table'; + id: string; + declaredRowCount?: number; + declaredColumnCount?: number; + rows: OneNoteTableRow[]; + columnWidths: number[]; + lockedColumns: boolean[]; + bordersVisible: boolean; + layout: OneNoteLayout; +} + +export interface OneNoteOutlineBlock { + kind: 'outline'; + id: string; + childLevel?: number; + indents: number[]; + layout: OneNoteLayout; + children: OneNoteContentBlock[]; +} + +export interface OneNoteOutlineElementBlock { + kind: 'outline-element'; + id: string; + childLevel?: number; + contents: OneNoteContentBlock[]; + children: OneNoteContentBlock[]; +} + +export interface OneNoteOutlineGroupBlock { + kind: 'outline-group'; + id: string; + childLevel?: number; + children: OneNoteContentBlock[]; +} + +export interface OneNoteInkBlock { + kind: 'ink'; + id: string; + supported: false; + description: string; +} + +export interface OneNoteUnsupportedBlock { + kind: 'unsupported'; + id: string; + sourceJcid: number; + description: string; +} + +export type OneNoteContentBlock = + | OneNoteTextBlock + | OneNoteImageBlock + | OneNoteAttachmentBlock + | OneNoteTableBlock + | OneNoteOutlineBlock + | OneNoteOutlineElementBlock + | OneNoteOutlineGroupBlock + | OneNoteInkBlock + | OneNoteUnsupportedBlock; + +export interface OneNoteResource { + id: string; + kind: 'image' | 'attachment'; + sourceJcid: number; + extension?: string; + filename?: string; + mediaType: string; + browserRenderable: boolean; + size?: number; + /** A bounded view descriptor. Bytes are not copied during parsing. */ + blob?: FileDataBlobReference; + availability: 'available' | 'external' | 'missing' | 'invalid'; +} + +export interface OneNotePageContentModel { + id: string; + title: string; + createdAt?: string; + updatedAt?: string; + level?: number; + text: string; + blocks: OneNoteContentBlock[]; +} + +/** Internal, non-serializable representation used by the worker integration. */ +export interface OneNoteSectionContentModel { + sourceName: string; + sectionName: string; + pages: OneNotePageContentModel[]; + resources: Map; + diagnostics: ParserDiagnostic[]; +} diff --git a/src/onenote/one/content.ts b/src/onenote/one/content.ts new file mode 100644 index 0000000..78822a0 --- /dev/null +++ b/src/onenote/one/content.ts @@ -0,0 +1,926 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +// +// Portions adapted from onenote.rs (MPL-2.0), revision +// 5138a39a3f4e72b840932f9872fecde52fa9da60: `onenote/{content,image, +// outline,rich_text,table,embedded_file}.rs`. Deviations: this module emits a +// browser-oriented, inert data model; links and renderable image formats are +// allow-listed; file payloads remain zero-copy references. + +import { BinaryReader } from '../binary/BinaryReader.js'; +import type { ParserDiagnostic } from '../model/diagnostics.js'; +import type { OneNoteParserLimits } from '../parser/limits.js'; +import type { + ExGuid, + FileDataBlobReference, + ObjectSpace, + StoreObject, +} from '../onestore/types.js'; +import { exGuidKey } from '../onestore/types.js'; +import type { + OneNoteAttachmentBlock, + OneNoteColor, + OneNoteContentBlock, + OneNoteImageBlock, + OneNoteLayout, + OneNoteResource, + OneNoteTableBlock, + OneNoteTextBlock, + OneNoteTextRun, + OneNoteTextStyle, +} from './content-model.js'; +import { JCID, PROPERTY } from './constants.js'; +import { + assertJcid, + getObject, + objectReferences, + optionalBool, + optionalBytes, + optionalF32, + optionalLatin1, + optionalString, + optionalU8, + optionalU16, + optionalU32, + optionalU32Vector, +} from './property-access.js'; + +export interface OneNoteContentBudget { + blocks: number; + textRuns: number; + tableCells: number; +} + +export interface OneNoteContentParseOptions { + space: ObjectSpace; + limits: OneNoteParserLimits; + diagnostics: ParserDiagnostic[]; + resources: Map; + pageId?: string; + budget?: OneNoteContentBudget; +} + +interface ContentContext extends OneNoteContentParseOptions { + budget: OneNoteContentBudget; +} + +export function createContentBudget(): OneNoteContentBudget { + return { blocks: 0, textRuns: 0, tableCells: 0 }; +} + +/** Parse page graph roots into an inert, recursively structured content tree. */ +export function parseContentObjects( + ids: ExGuid[], + options: OneNoteContentParseOptions +): OneNoteContentBlock[] { + const context: ContentContext = { + ...options, + budget: options.budget ?? createContentBudget(), + }; + const result: OneNoteContentBlock[] = []; + for (const id of ids) { + const block = parseSafely(context, id, new Set(), 0); + if (block) result.push(block); + } + return result; +} + +export function contentBlocksToPlainText( + blocks: readonly OneNoteContentBlock[] +): string { + return blocks + .map(blockToPlainText) + .filter((value) => value.length > 0) + .join('\n'); +} + +function parseSafely( + context: ContentContext, + id: ExGuid, + active: Set, + depth: number +): OneNoteContentBlock | undefined { + try { + return parseObject(context, id, active, depth); + } catch (error) { + addDiagnostic( + context, + 'CONTENT_PARSE_FAILED', + `Could not parse content ${exGuidKey(id)}: ${error instanceof Error ? error.message : String(error)}`, + 'MS-ONE content graph' + ); + return undefined; + } +} + +function parseObject( + context: ContentContext, + id: ExGuid, + active: Set, + depth: number +): OneNoteContentBlock { + if (depth > context.limits.maxGraphDepth) { + throw new Error( + `Content graph depth exceeds ${context.limits.maxGraphDepth}` + ); + } + incrementBlockBudget(context); + const key = exGuidKey(id); + if (active.has(key)) throw new Error(`Content graph cycle at ${key}`); + const object = getObject(context.space, id, 'content'); + + active.add(key); + try { + switch (object.jcid) { + case JCID.OutlineNode: + return { + kind: 'outline', + id: key, + childLevel: optionalU8(object, PROPERTY.OutlineElementChildLevel), + indents: parseFloatVector( + optionalBytes(object, PROPERTY.RgOutlineIndentDistance) + ), + layout: parseLayout(object), + children: parseChildren( + context, + objectReferences(object, PROPERTY.ElementChildNodes), + active, + depth + ), + }; + case JCID.OutlineGroup: + return { + kind: 'outline-group', + id: key, + childLevel: optionalU8(object, PROPERTY.OutlineElementChildLevel), + children: parseChildren( + context, + objectReferences(object, PROPERTY.ElementChildNodes), + active, + depth + ), + }; + case JCID.OutlineElementNode: + return { + kind: 'outline-element', + id: key, + childLevel: optionalU8(object, PROPERTY.OutlineElementChildLevel), + contents: parseChildren( + context, + objectReferences(object, PROPERTY.ContentChildNodes), + active, + depth + ), + children: parseChildren( + context, + objectReferences(object, PROPERTY.ElementChildNodes), + active, + depth + ), + }; + case JCID.RichTextNode: + return parseRichText(context, object); + case JCID.ImageNode: + return parseImage(context, object); + case JCID.EmbeddedFileNode: + return parseAttachment(context, object); + case JCID.TableNode: + return parseTable(context, object, active, depth); + case JCID.InkContainer: + addDiagnostic( + context, + 'UNSUPPORTED_INK', + 'Ink content is retained as a typed placeholder', + `Object ${key}` + ); + return { + kind: 'ink', + id: key, + supported: false, + description: 'Ink parsing is not implemented in this slice', + }; + default: + addDiagnostic( + context, + 'UNSUPPORTED_CONTENT_OBJECT', + `Content object JCID 0x${object.jcid.toString(16)} is not rendered`, + `Object ${key}` + ); + return { + kind: 'unsupported', + id: key, + sourceJcid: object.jcid, + description: `Unsupported OneNote object 0x${object.jcid.toString(16)}`, + }; + } + } finally { + active.delete(key); + } +} + +function parseChildren( + context: ContentContext, + ids: ExGuid[], + active: Set, + depth: number +): OneNoteContentBlock[] { + const result: OneNoteContentBlock[] = []; + for (const id of ids) { + const block = parseSafely(context, id, active, depth + 1); + if (block) result.push(block); + } + return result; +} + +function parseRichText( + context: ContentContext, + object: StoreObject +): OneNoteTextBlock { + let sourceText = optionalString(object, PROPERTY.RichEditTextUnicode); + sourceText ??= optionalLatin1(object, PROPERTY.TextExtendedAscii); + sourceText = (sourceText ?? '').split('\0').join(''); + + const paragraphStyleId = objectReferences(object, PROPERTY.ParagraphStyle)[0]; + const paragraphStyle = paragraphStyleId + ? parseStyleReference(context, paragraphStyleId) + : {}; + const styleIds = objectReferences(object, PROPERTY.TextRunFormatting); + let indices = optionalU32Vector(object, PROPERTY.TextRunIndex) ?? []; + let styles = styleIds.map((id) => parseStyleReference(context, id)); + ({ sourceText, indices, styles } = fixLeadingVerticalTab( + sourceText, + indices, + styles + )); + validateTextRuns(context, sourceText, indices, styles); + + const runs = createTextRuns(sourceText, indices, styles, paragraphStyle); + incrementTextRunBudget(context, runs.length); + return { + kind: 'text', + id: exGuidKey(object.id), + sourceText, + text: runs + .filter((run) => run.style.hidden !== true) + .map((run) => run.text) + .join(''), + runs, + paragraphStyle, + paragraphAlignment: paragraphAlignment( + optionalU8(object, PROPERTY.ParagraphAlignment) + ), + paragraphSpaceBefore: optionalF32(object, PROPERTY.ParagraphSpaceBefore), + paragraphSpaceAfter: optionalF32(object, PROPERTY.ParagraphSpaceAfter), + paragraphLineSpacingExact: optionalF32( + object, + PROPERTY.ParagraphLineSpacingExact + ), + rightToLeft: optionalBool(object, PROPERTY.ReadingOrderRtl), + layout: parseLayout(object), + }; +} + +function validateTextRuns( + context: ContentContext, + text: string, + indices: readonly number[], + styles: readonly OneNoteTextStyle[] +): void { + if (styles.length > context.limits.maxTextRuns) { + throw new Error(`Text run count exceeds ${context.limits.maxTextRuns}`); + } + if (styles.length === 0) { + if (indices.length !== 0) { + throw new Error('TextRunIndex is present without text-run formatting'); + } + return; + } + if (styles.length !== indices.length + 1) { + if (text.length === 0 && indices.length === 0) return; + throw new Error( + `Text-run style/index mismatch (${styles.length} styles, ${indices.length} indices)` + ); + } + let previous = 0; + for (const index of indices) { + if ( + !Number.isSafeInteger(index) || + index <= previous || + index > text.length + ) { + throw new Error( + `Invalid TextRunIndex ${index} for text length ${text.length}` + ); + } + previous = index; + } +} + +function createTextRuns( + text: string, + indices: readonly number[], + styles: readonly OneNoteTextStyle[], + paragraphStyle: OneNoteTextStyle +): OneNoteTextRun[] { + if (text.length === 0) return []; + if (styles.length === 0) { + return [{ start: 0, end: text.length, text, style: paragraphStyle }]; + } + + const boundaries = [0, ...indices, text.length]; + const result: OneNoteTextRun[] = []; + let pendingHref: string | undefined; + for (let index = 0; index < styles.length; index += 1) { + const start = boundaries[index]!; + const end = boundaries[index + 1]!; + const runText = text.slice(start, end); + const style = mergeStyles(paragraphStyle, styles[index]!); + const markerHref = style.hidden ? hyperlinkMarker(runText) : undefined; + if (markerHref !== undefined) pendingHref = safeHref(markerHref); + + const run: OneNoteTextRun = { start, end, text: runText, style }; + if ( + !style.hidden && + pendingHref && + (style.hyperlinkProtected === true || style.hyperlink === true) + ) { + run.href = pendingHref; + pendingHref = undefined; + } + result.push(run); + } + return result; +} + +function parseStyleReference( + context: ContentContext, + id: ExGuid +): OneNoteTextStyle { + const object = context.space.objects.get(exGuidKey(id)); + if (!object) { + addDiagnostic( + context, + 'MISSING_TEXT_STYLE', + `Text style ${exGuidKey(id)} is missing`, + 'RichTextNode' + ); + return {}; + } + if (object.jcid !== JCID.ParagraphStyleObject) { + addDiagnostic( + context, + 'INVALID_TEXT_STYLE', + `Style ${exGuidKey(id)} has JCID 0x${object.jcid.toString(16)}`, + 'RichTextNode' + ); + return {}; + } + return compactObject({ + charset: optionalU8(object, PROPERTY.Charset), + bold: optionalBool(object, PROPERTY.Bold), + italic: optionalBool(object, PROPERTY.Italic), + underline: optionalBool(object, PROPERTY.Underline), + strikethrough: optionalBool(object, PROPERTY.Strikethrough), + superscript: optionalBool(object, PROPERTY.Superscript), + subscript: optionalBool(object, PROPERTY.Subscript), + font: cleanMetadataString(optionalString(object, PROPERTY.Font)), + fontSize: optionalU16(object, PROPERTY.FontSize), + fontColor: parseColorRef(optionalU32(object, PROPERTY.FontColor)), + highlight: parseColorRef(optionalU32(object, PROPERTY.Highlight)), + languageCode: optionalU32(object, PROPERTY.LanguageId), + mathFormatting: optionalBool(object, PROPERTY.MathFormatting), + hyperlink: optionalBool(object, PROPERTY.Hyperlink), + hyperlinkProtected: optionalBool(object, PROPERTY.HyperlinkProtected), + hidden: optionalBool(object, PROPERTY.Hidden), + }); +} + +function parseImage( + context: ContentContext, + object: StoreObject +): OneNoteImageBlock { + const containerId = + objectReferences(object, PROPERTY.PictureContainer)[0] ?? + objectReferences(object, PROPERTY.WebPictureContainer14)[0]; + const filename = cleanMetadataString( + optionalString(object, PROPERTY.ImageFilename) + ); + const resourceId = containerId + ? registerResource(context, containerId, 'image', filename) + : undefined; + return compactObject({ + kind: 'image' as const, + id: exGuidKey(object.id), + resourceId, + filename, + altText: cleanMetadataString(optionalString(object, PROPERTY.ImageAltText)), + ocrText: cleanMetadataString( + optionalString(object, PROPERTY.RichEditTextUnicode) + ), + pictureWidth: optionalF32(object, PROPERTY.PictureWidth), + pictureHeight: optionalF32(object, PROPERTY.PictureHeight), + displayedPageNumber: optionalU32(object, PROPERTY.DisplayedPageNumber), + href: safeHref(optionalString(object, PROPERTY.WzHyperlinkUrl)), + isBackground: optionalBool(object, PROPERTY.IsBackground) ?? false, + layout: parseLayout(object), + }); +} + +function parseAttachment( + context: ContentContext, + object: StoreObject +): OneNoteAttachmentBlock { + const filename = + cleanMetadataString(optionalString(object, PROPERTY.EmbeddedFileName)) ?? + 'Unnamed attachment'; + const containerId = objectReferences( + object, + PROPERTY.EmbeddedFileContainer + )[0]; + const iconId = objectReferences(object, PROPERTY.PictureContainer)[0]; + const resourceId = containerId + ? registerResource(context, containerId, 'attachment', filename) + : undefined; + const iconResourceId = iconId + ? registerResource(context, iconId, 'image') + : undefined; + return compactObject({ + kind: 'attachment' as const, + id: exGuidKey(object.id), + resourceId, + iconResourceId, + filename, + size: resourceId ? context.resources.get(resourceId)?.size : undefined, + layout: parseLayout(object), + }); +} + +function registerResource( + context: ContentContext, + id: ExGuid, + kind: OneNoteResource['kind'], + filename?: string +): string | undefined { + const key = exGuidKey(id); + const existing = context.resources.get(key); + if (existing) { + if (!existing.filename && filename) existing.filename = filename; + return key; + } + const object = context.space.objects.get(key); + if (!object) { + addDiagnostic( + context, + 'MISSING_RESOURCE_CONTAINER', + `Resource container ${key} is missing`, + 'File data' + ); + return undefined; + } + const expected = + kind === 'attachment' + ? object.jcid === JCID.EmbeddedFileContainer + : object.jcid === JCID.PictureContainer || + object.jcid === JCID.XpsContainer; + if (!expected) { + addDiagnostic( + context, + 'INVALID_RESOURCE_CONTAINER', + `Resource ${key} has unexpected JCID 0x${object.jcid.toString(16)}`, + 'File data' + ); + return undefined; + } + + const extension = normalizeExtension( + object.fileData?.extension ?? extensionFromFilename(filename) + ); + const sniffed = sniffResource(object.fileData?.blob, extension); + const availability = resourceAvailability(object); + context.resources.set(key, { + id: key, + kind, + sourceJcid: object.jcid, + extension, + filename, + mediaType: sniffed.mediaType, + browserRenderable: kind === 'image' && sniffed.browserRenderable, + size: object.fileData?.blob?.size, + blob: object.fileData?.blob, + availability, + }); + return key; +} + +function parseTable( + context: ContentContext, + object: StoreObject, + active: Set, + depth: number +): OneNoteTableBlock { + const declaredRowCount = optionalU32(object, PROPERTY.RowCount); + const declaredColumnCount = optionalU32(object, PROPERTY.ColumnCount); + const rowIds = objectReferences(object, PROPERTY.ElementChildNodes); + if (declaredRowCount !== undefined && declaredRowCount !== rowIds.length) { + addDiagnostic( + context, + 'TABLE_ROW_COUNT_MISMATCH', + `Table declares ${declaredRowCount} rows but references ${rowIds.length}`, + `Table ${exGuidKey(object.id)}` + ); + } + + const rows = rowIds.map((rowId) => { + const row = getObject(context.space, rowId, 'table row'); + assertJcid(row, JCID.TableRowNode, 'TableRowNode'); + const cellIds = objectReferences(row, PROPERTY.ElementChildNodes); + if ( + declaredColumnCount !== undefined && + cellIds.length !== declaredColumnCount + ) { + addDiagnostic( + context, + 'TABLE_COLUMN_COUNT_MISMATCH', + `Table row has ${cellIds.length} cells; expected ${declaredColumnCount}`, + `TableRow ${exGuidKey(row.id)}` + ); + } + return { + id: exGuidKey(row.id), + cells: cellIds.map((cellId) => { + incrementTableCellBudget(context); + const cell = getObject(context.space, cellId, 'table cell'); + assertJcid(cell, JCID.TableCellNode, 'TableCellNode'); + return { + id: exGuidKey(cell.id), + blocks: parseChildren( + context, + objectReferences(cell, PROPERTY.ElementChildNodes), + active, + depth + ), + backgroundColor: parseColor( + optionalU32(cell, PROPERTY.CellBackgroundColor) + ), + maxWidth: optionalF32(cell, PROPERTY.LayoutMaxWidth), + }; + }), + }; + }); + + return { + kind: 'table', + id: exGuidKey(object.id), + declaredRowCount, + declaredColumnCount, + rows, + columnWidths: parseCountedFloatVector( + optionalBytes(object, PROPERTY.TableColumnWidths), + context, + 'TableColumnWidths' + ), + lockedColumns: parseLockedColumns( + optionalBytes(object, PROPERTY.TableColumnsLocked), + declaredColumnCount, + context + ), + bordersVisible: optionalBool(object, PROPERTY.TableBordersVisible) ?? true, + layout: parseLayout(object), + }; +} + +function parseLayout(object: StoreObject): OneNoteLayout { + return compactObject({ + maxWidth: optionalF32(object, PROPERTY.LayoutMaxWidth), + maxHeight: optionalF32(object, PROPERTY.LayoutMaxHeight), + offsetHorizontal: optionalF32(object, PROPERTY.OffsetFromParentHoriz), + offsetVertical: optionalF32(object, PROPERTY.OffsetFromParentVert), + alignmentInParent: optionalU32(object, PROPERTY.LayoutAlignmentInParent), + alignmentSelf: optionalU32(object, PROPERTY.LayoutAlignmentSelf), + sizeSetByUser: optionalBool(object, PROPERTY.IsLayoutSizeSetByUser), + }); +} + +function parseCountedFloatVector( + bytes: Uint8Array | undefined, + context: ContentContext, + structure: string +): number[] { + if (!bytes || bytes.length === 0) return []; + const count = bytes[0]!; + const expected = 1 + count * 4; + if (bytes.length !== expected) { + addDiagnostic( + context, + 'INVALID_TABLE_VECTOR', + `${structure} has ${bytes.length} bytes; expected ${expected}`, + structure + ); + } + const available = Math.min(count, Math.floor((bytes.length - 1) / 4)); + const reader = new BinaryReader(bytes, 1, available * 4, structure); + const result: number[] = []; + for (let index = 0; index < available; index += 1) { + const raw = reader.u32(); + const data = new DataView(new ArrayBuffer(4)); + data.setUint32(0, raw, true); + const value = data.getFloat32(0, true); + if (Number.isFinite(value)) result.push(value); + } + return result; +} + +function parseFloatVector(bytes: Uint8Array | undefined): number[] { + if (!bytes || bytes.length === 0) return []; + const count = bytes[0]!; + const available = Math.min(count, Math.floor((bytes.length - 1) / 4)); + const reader = new BinaryReader(bytes, 1, available * 4, 'float vector'); + const result: number[] = []; + for (let index = 0; index < available; index += 1) { + const raw = reader.u32(); + const data = new DataView(new ArrayBuffer(4)); + data.setUint32(0, raw, true); + const value = data.getFloat32(0, true); + if (Number.isFinite(value)) result.push(value); + } + return result; +} + +function parseLockedColumns( + bytes: Uint8Array | undefined, + columnCount: number | undefined, + context: ContentContext +): boolean[] { + if (!bytes || bytes.length === 0) return []; + const bitCount = bytes[0]!; + const byteCount = Math.ceil(bitCount / 8); + if (bytes.length !== byteCount + 1) { + addDiagnostic( + context, + 'INVALID_TABLE_VECTOR', + `TableColumnsLocked has ${bytes.length} bytes; expected ${byteCount + 1}`, + 'TableColumnsLocked' + ); + } + const bitmap = bytes.subarray(1, Math.min(bytes.length, byteCount + 1)); + const count = columnCount ?? bitCount; + return Array.from( + { length: count }, + (_, index) => + (((bitmap[Math.floor(index / 8)] ?? 0) >>> (index % 8)) & 1) !== 0 + ); +} + +function sniffResource( + blob: FileDataBlobReference | undefined, + extension: string | undefined +): { mediaType: string; browserRenderable: boolean } { + if (blob && blob.size >= 8) { + const bytes = blob.source.subarray(blob.offset, blob.offset + 12); + if ( + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 && + bytes[4] === 0x0d && + bytes[5] === 0x0a && + bytes[6] === 0x1a && + bytes[7] === 0x0a + ) { + return { mediaType: 'image/png', browserRenderable: true }; + } + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return { mediaType: 'image/jpeg', browserRenderable: true }; + } + } + return { + mediaType: mediaTypeFromExtension(extension), + // Extension metadata is untrusted; only a recognized signature is shown. + browserRenderable: false, + }; +} + +function mediaTypeFromExtension(extension: string | undefined): string { + switch (extension) { + case 'png': + return 'image/png'; + case 'jpg': + case 'jpeg': + return 'image/jpeg'; + case 'emf': + return 'image/emf'; + case 'wmf': + return 'image/wmf'; + case 'pdf': + return 'application/pdf'; + case 'docx': + return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; + case 'xlsx': + return 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + case 'pptx': + return 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; + case 'txt': + return 'text/plain'; + default: + return 'application/octet-stream'; + } +} + +function resourceAvailability( + object: StoreObject +): OneNoteResource['availability'] { + if (!object.fileData) return 'missing'; + if (object.fileData.referenceKind === 'invalid') return 'invalid'; + if (object.fileData.referenceKind === 'external') return 'external'; + return object.fileData.blob ? 'available' : 'missing'; +} + +function normalizeExtension(value: string | undefined): string | undefined { + const normalized = value + ?.split('\0') + .join('') + .trim() + .replace(/^\.+/, '') + .toLowerCase(); + return normalized || undefined; +} + +function extensionFromFilename( + filename: string | undefined +): string | undefined { + if (!filename) return undefined; + const basename = filename.split(/[\\/]/).at(-1) ?? filename; + const index = basename.lastIndexOf('.'); + return index < 0 ? undefined : basename.slice(index + 1); +} + +function cleanMetadataString(value: string | undefined): string | undefined { + const cleaned = value?.split('\0').join('').trim(); + return cleaned || undefined; +} + +function safeHref(value: string | undefined): string | undefined { + const candidate = cleanMetadataString(value); + if (!candidate || containsControlCharacter(candidate)) return undefined; + try { + const url = new URL(candidate); + return url.protocol === 'http:' || + url.protocol === 'https:' || + url.protocol === 'mailto:' + ? candidate + : undefined; + } catch { + return undefined; + } +} + +function containsControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 0x1f || code === 0x7f) return true; + } + return false; +} + +function hyperlinkMarker(text: string): string | undefined { + return /\ufddfHYPERLINK\s+"([^"]+)"/iu.exec(text)?.[1]; +} + +function parseColorRef(value: number | undefined): OneNoteColor | undefined { + if (value === undefined) return undefined; + const marker = value >>> 24; + if (marker === 0xff) return { kind: 'auto' }; + if (marker !== 0) return undefined; + return { + kind: 'rgb', + red: value & 0xff, + green: (value >>> 8) & 0xff, + blue: (value >>> 16) & 0xff, + }; +} + +function parseColor(value: number | undefined): OneNoteColor | undefined { + if (value === undefined) return undefined; + const alpha = 255 - ((value >>> 24) & 0xff); + return { + kind: alpha === 255 ? 'rgb' : 'rgba', + red: value & 0xff, + green: (value >>> 8) & 0xff, + blue: (value >>> 16) & 0xff, + ...(alpha === 255 ? {} : { alpha }), + }; +} + +function paragraphAlignment( + value: number | undefined +): OneNoteTextBlock['paragraphAlignment'] { + if (value === undefined) return undefined; + if (value === 0) return 'left'; + if (value === 1) return 'center'; + if (value === 2) return 'right'; + return 'unknown'; +} + +function mergeStyles( + base: OneNoteTextStyle, + override: OneNoteTextStyle +): OneNoteTextStyle { + return compactObject({ ...base, ...override }); +} + +function fixLeadingVerticalTab( + sourceText: string, + indices: number[], + styles: OneNoteTextStyle[] +): { + sourceText: string; + indices: number[]; + styles: OneNoteTextStyle[]; +} { + if (!sourceText.startsWith('\u000b') || indices[0] !== 1) { + return { sourceText, indices, styles }; + } + const difference = styles.length - indices.length; + if (difference !== 0 && difference !== 1) { + return { sourceText, indices, styles }; + } + sourceText = sourceText.slice(1); + indices = indices.slice(1).map((index) => index - 1); + if (difference === 1) styles = styles.slice(0, -1); + return { sourceText, indices, styles }; +} + +function blockToPlainText(block: OneNoteContentBlock): string { + switch (block.kind) { + case 'text': + return block.text.split('\u000b').join('\n').trimEnd(); + case 'outline': + case 'outline-group': + return contentBlocksToPlainText(block.children); + case 'outline-element': { + const contents = contentBlocksToPlainText(block.contents); + const children = contentBlocksToPlainText(block.children); + return [contents, children].filter(Boolean).join('\n'); + } + case 'table': + return block.rows + .map((row) => + row.cells + .map((cell) => contentBlocksToPlainText(cell.blocks)) + .join('\t') + ) + .join('\n'); + default: + return ''; + } +} + +function incrementBlockBudget(context: ContentContext): void { + context.budget.blocks += 1; + if (context.budget.blocks > context.limits.maxContentBlocks) { + throw new Error( + `Content block count exceeds ${context.limits.maxContentBlocks}` + ); + } +} + +function incrementTextRunBudget(context: ContentContext, amount: number): void { + context.budget.textRuns += amount; + if (context.budget.textRuns > context.limits.maxTextRuns) { + throw new Error(`Text run count exceeds ${context.limits.maxTextRuns}`); + } +} + +function incrementTableCellBudget(context: ContentContext): void { + context.budget.tableCells += 1; + if (context.budget.tableCells > context.limits.maxTableCells) { + throw new Error(`Table cell count exceeds ${context.limits.maxTableCells}`); + } +} + +function addDiagnostic( + context: ContentContext, + code: string, + message: string, + structure: string +): void { + if (context.diagnostics.length >= context.limits.maxDiagnostics) return; + context.diagnostics.push({ + severity: 'warning', + code, + message, + structure: context.pageId + ? `${structure} (page ${context.pageId})` + : structure, + recoverable: true, + }); +} + +function compactObject(value: T): T { + for (const key of Object.keys(value) as (keyof T)[]) { + if (value[key] === undefined) delete value[key]; + } + return value; +} diff --git a/src/onenote/one/property-access.ts b/src/onenote/one/property-access.ts new file mode 100644 index 0000000..fc93c5d --- /dev/null +++ b/src/onenote/one/property-access.ts @@ -0,0 +1,262 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +// +// Portions adapted from onenote.rs (MPL-2.0), revision +// 5138a39a3f4e72b840932f9872fecde52fa9da60: `one/property/{simple, +// object_reference}.rs`. Deviations: values stay in the TypeScript OneStore +// model and malformed property types are rejected rather than coerced. + +import { BinaryReader } from '../binary/BinaryReader.js'; +import { readGuid } from '../binary/guid.js'; +import { + countObjectReferences, + countObjectSpaceReferences, + findProperty, + referenceOffset, +} from '../onestore/property-set.js'; +import type { + ExGuid, + ObjectSpace, + PropertyEntry, + StoreObject, +} from '../onestore/types.js'; +import { exGuidKey } from '../onestore/types.js'; + +export function objectReferences( + object: StoreObject, + propertyId: number +): ExGuid[] { + const property = findProperty(object.props, propertyId); + if (!property) return []; + let count: number; + if (property.value.kind === 'objectId') count = 1; + else if (property.value.kind === 'objectIds') count = property.value.count; + else throw propertyTypeError(propertyId, 'an object reference'); + + const offset = referenceOffset( + object.props, + propertyId, + countObjectReferences + ); + return resolveReferenceSlice(object, object.props.objectIds, offset, count); +} + +export function objectSpaceReferences( + object: StoreObject, + propertyId: number +): ExGuid[] { + const property = findProperty(object.props, propertyId); + if (!property) return []; + let count: number; + if (property.value.kind === 'objectSpaceId') count = 1; + else if (property.value.kind === 'objectSpaceIds') + count = property.value.count; + else throw propertyTypeError(propertyId, 'an object-space reference'); + + const offset = referenceOffset( + object.props, + propertyId, + countObjectSpaceReferences + ); + return resolveReferenceSlice( + object, + object.props.objectSpaceIds, + offset, + count + ); +} + +export function rootObject( + space: ObjectSpace, + role: number, + structure: string +): StoreObject { + const id = space.roots.get(role); + if (!id) throw new Error(`Object space has no ${structure}`); + return getObject(space, id, structure); +} + +export function getObject( + space: ObjectSpace, + id: ExGuid, + structure: string +): StoreObject { + const object = space.objects.get(exGuidKey(id)); + if (!object) + throw new Error(`${structure} object ${exGuidKey(id)} is missing`); + return object; +} + +export function assertJcid( + object: StoreObject, + expected: number, + structure: string +): void { + if (object.jcid !== expected) { + throw new Error( + `${structure} has JCID 0x${object.jcid.toString(16)}, expected 0x${expected.toString(16)}` + ); + } +} + +export function optionalEntry( + object: StoreObject, + propertyId: number +): PropertyEntry | undefined { + return findProperty(object.props, propertyId); +} + +export function optionalBytes( + object: StoreObject, + propertyId: number +): Uint8Array | undefined { + const value = optionalEntry(object, propertyId)?.value; + if (!value) return undefined; + if (value.kind !== 'bytes') throw propertyTypeError(propertyId, 'bytes'); + return value.value; +} + +export function optionalString( + object: StoreObject, + propertyId: number +): string | undefined { + const bytes = optionalBytes(object, propertyId); + if (!bytes) return undefined; + if (bytes.byteLength % 2 !== 0) { + throw new Error( + `UTF-16 property 0x${propertyId.toString(16)} has odd length` + ); + } + return new TextDecoder('utf-16le', { fatal: true }).decode(bytes); +} + +export function optionalLatin1( + object: StoreObject, + propertyId: number +): string | undefined { + const bytes = optionalBytes(object, propertyId); + if (!bytes) return undefined; + let result = ''; + for (const value of bytes) result += String.fromCharCode(value); + return result; +} + +export function requiredGuid( + object: StoreObject, + propertyId: number, + structure: string +): string { + const bytes = optionalBytes(object, propertyId); + if (!bytes || bytes.length !== 16) { + throw new Error(`${structure} is absent or not 16 bytes`); + } + return readGuid(new BinaryReader(bytes, 0, 16, structure)); +} + +export function optionalBool( + object: StoreObject, + propertyId: number +): boolean | undefined { + const value = optionalEntry(object, propertyId)?.value; + if (!value) return undefined; + if (value.kind !== 'bool') throw propertyTypeError(propertyId, 'a Boolean'); + return value.value; +} + +export function optionalU8( + object: StoreObject, + propertyId: number +): number | undefined { + const value = optionalEntry(object, propertyId)?.value; + if (!value) return undefined; + if (value.kind !== 'u8') throw propertyTypeError(propertyId, 'a u8'); + return value.value; +} + +export function optionalU16( + object: StoreObject, + propertyId: number +): number | undefined { + const value = optionalEntry(object, propertyId)?.value; + if (!value) return undefined; + if (value.kind !== 'u16') throw propertyTypeError(propertyId, 'a u16'); + return value.value; +} + +export function optionalU32( + object: StoreObject, + propertyId: number +): number | undefined { + const value = optionalEntry(object, propertyId)?.value; + if (!value) return undefined; + if (value.kind !== 'u32') throw propertyTypeError(propertyId, 'a u32'); + return value.value; +} + +export function optionalU64( + object: StoreObject, + propertyId: number +): bigint | undefined { + const value = optionalEntry(object, propertyId)?.value; + if (!value) return undefined; + if (value.kind !== 'u64') throw propertyTypeError(propertyId, 'a u64'); + return value.value; +} + +export function optionalF32( + object: StoreObject, + propertyId: number +): number | undefined { + const raw = optionalU32(object, propertyId); + if (raw === undefined) return undefined; + const data = new DataView(new ArrayBuffer(4)); + data.setUint32(0, raw, true); + const value = data.getFloat32(0, true); + if (!Number.isFinite(value)) { + throw new Error( + `Float property 0x${propertyId.toString(16)} is not finite` + ); + } + return value; +} + +export function optionalU32Vector( + object: StoreObject, + propertyId: number +): number[] | undefined { + const bytes = optionalBytes(object, propertyId); + if (!bytes) return undefined; + if (bytes.byteLength % 4 !== 0) { + throw new Error( + `Property 0x${propertyId.toString(16)} is not a u32 vector` + ); + } + const reader = new BinaryReader(bytes, 0, bytes.byteLength, 'u32 vector'); + const result: number[] = []; + while (reader.remaining > 0) result.push(reader.u32()); + return result; +} + +function resolveReferenceSlice( + object: StoreObject, + ids: { guidIndex: number; value: number }[], + offset: number, + count: number +): ExGuid[] { + if (offset + count > ids.length) { + throw new Error( + `Reference range ${offset}+${count} exceeds stream length ${ids.length}` + ); + } + return ids.slice(offset, offset + count).map((id) => { + const guid = object.mapping.get(id.guidIndex); + if (!guid) throw new Error(`GUID mapping ${id.guidIndex} is missing`); + return { guid, value: id.value }; + }); +} + +function propertyTypeError(propertyId: number, expected: string): Error { + return new Error(`Property 0x${propertyId.toString(16)} is not ${expected}`); +} diff --git a/src/onenote/one/section-content.ts b/src/onenote/one/section-content.ts new file mode 100644 index 0000000..c238855 --- /dev/null +++ b/src/onenote/one/section-content.ts @@ -0,0 +1,294 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +// +// Portions adapted from onenote.rs (MPL-2.0), revision +// 5138a39a3f4e72b840932f9872fecde52fa9da60: `onenote/{section, +// page_series,page}.rs`. Deviations: pages retain the internal content/resource +// model needed for incremental worker integration. + +import { filetimeToIso, oneNoteTimeToIso } from '../binary/time.js'; +import type { ParserDiagnostic } from '../model/diagnostics.js'; +import type { DesktopOneStore } from '../onestore/desktop-store.js'; +import type { ObjectSpace } from '../onestore/types.js'; +import { exGuidKey } from '../onestore/types.js'; +import type { OneNoteParserLimits } from '../parser/limits.js'; +import { + contentBlocksToPlainText, + createContentBudget, + parseContentObjects, +} from './content.js'; +import type { + OneNotePageContentModel, + OneNoteResource, + OneNoteSectionContentModel, +} from './content-model.js'; +import { JCID, PROPERTY } from './constants.js'; +import { + assertJcid, + getObject, + objectReferences, + objectSpaceReferences, + optionalString, + optionalU32, + optionalU64, + requiredGuid, + rootObject, +} from './property-access.js'; + +interface SectionContentContext { + store: DesktopOneStore; + limits: OneNoteParserLimits; + diagnostics: ParserDiagnostic[]; + resources: Map; + budget: ReturnType; + extractedChars: number; + currentPage?: string; +} + +export function buildSectionContentModel( + store: DesktopOneStore, + sourceName: string, + limits: OneNoteParserLimits, + diagnostics: ParserDiagnostic[] +): OneNoteSectionContentModel { + if (store.kind !== 'section') { + throw new Error('A table-of-contents store cannot be parsed as a section'); + } + const context: SectionContentContext = { + store, + limits, + diagnostics, + resources: new Map(), + budget: createContentBudget(), + extractedChars: 0, + }; + const root = store.rootObjectSpace; + const metadata = rootObject(root, 2, 'section metadata root'); + assertJcid(metadata, JCID.SectionMetadata, 'SectionMetadata'); + const displayName = cleanMetadataString( + optionalString(metadata, PROPERTY.SectionDisplayName) + ); + + const section = rootObject(root, 1, 'section content root'); + assertJcid(section, JCID.SectionNode, 'SectionNode'); + const pageSeriesIds = objectReferences(section, PROPERTY.ElementChildNodes); + const pages: OneNotePageContentModel[] = []; + + for (const seriesId of pageSeriesIds) { + const series = getObject(root, seriesId, 'PageSeriesNode'); + assertJcid(series, JCID.PageSeriesNode, 'PageSeriesNode'); + const pageSpaceIds = objectSpaceReferences( + series, + PROPERTY.ChildGraphSpaceElementNodes + ); + for (const pageSpaceId of pageSpaceIds) { + context.currentPage = undefined; + const pageSpace = store.objectSpaces.get(exGuidKey(pageSpaceId)); + if (!pageSpace) { + addDiagnostic( + context, + 'MISSING_PAGE_SPACE', + `Page object space ${exGuidKey(pageSpaceId)} is missing`, + 'PageSeriesNode' + ); + continue; + } + if (pages.length >= limits.maxPages) { + addDiagnostic( + context, + 'PAGE_LIMIT_REACHED', + `Page count exceeds ${limits.maxPages}`, + 'PageSeriesNode', + 'error' + ); + break; + } + try { + pages.push(parsePage(context, pageSpace)); + } catch (error) { + addDiagnostic( + context, + 'PAGE_PARSE_FAILED', + `Could not parse page ${exGuidKey(pageSpaceId)}: ${error instanceof Error ? error.message : String(error)}`, + 'Page', + 'error' + ); + } + } + if (pages.length >= limits.maxPages) break; + } + + return { + sourceName, + sectionName: displayName ?? stripOneExtension(sourceName), + pages, + resources: context.resources, + diagnostics, + }; +} + +function parsePage( + context: SectionContentContext, + space: ObjectSpace +): OneNotePageContentModel { + const metadata = rootObject(space, 2, 'page metadata root'); + assertJcid(metadata, JCID.PageMetadata, 'PageMetadata'); + const pageGuid = requiredGuid( + metadata, + PROPERTY.NotebookManagementEntityGuid, + 'PageMetadata entity GUID' + ); + context.currentPage = pageGuid; + const created = optionalU64(metadata, PROPERTY.TopologyCreationTimeStamp); + const level = optionalU32(metadata, PROPERTY.PageLevel); + const cachedMetadataTitle = cleanMetadataString( + optionalString(metadata, PROPERTY.CachedTitleString) + ); + + const manifest = rootObject(space, 1, 'page content root'); + assertJcid(manifest, JCID.PageManifestNode, 'PageManifestNode'); + const pageId = objectReferences(manifest, PROPERTY.ContentChildNodes)[0]; + if (!pageId) throw new Error('PageManifestNode has no page reference'); + const page = getObject(space, pageId, 'PageNode'); + assertJcid(page, JCID.PageNode, 'PageNode'); + const updated = optionalU32(page, PROPERTY.LastModifiedTime); + const cachedPageTitle = cleanMetadataString( + optionalString(page, PROPERTY.CachedTitleStringFromPage) + ); + + const contentOptions = { + space, + limits: context.limits, + diagnostics: context.diagnostics, + resources: context.resources, + pageId: pageGuid, + budget: context.budget, + }; + const titleId = objectReferences( + page, + PROPERTY.StructureElementChildNodes + )[0]; + let titleText: string | undefined; + if (titleId) { + try { + const title = getObject(space, titleId, 'TitleNode'); + assertJcid(title, JCID.TitleNode, 'TitleNode'); + const titleBlocks = parseContentObjects( + objectReferences(title, PROPERTY.ElementChildNodes), + contentOptions + ); + titleText = cleanTitle(contentBlocksToPlainText(titleBlocks)); + } catch (error) { + addDiagnostic( + context, + 'TITLE_PARSE_FAILED', + error instanceof Error ? error.message : String(error), + 'TitleNode' + ); + } + } + + const blocks = parseContentObjects( + objectReferences(page, PROPERTY.ElementChildNodes), + contentOptions + ); + let text = contentBlocksToPlainText(blocks); + const remaining = + context.limits.maxExtractedTextChars - context.extractedChars; + if (text.length > remaining) { + text = text.slice(0, Math.max(0, remaining)); + addDiagnostic( + context, + 'TEXT_LIMIT_REACHED', + `Extracted text was truncated at ${context.limits.maxExtractedTextChars} characters`, + `Page ${pageGuid}` + ); + } + context.extractedChars += text.length; + + const title = + titleText || + firstNonemptyLine(text) || + cachedPageTitle || + cachedMetadataTitle || + 'Untitled page'; + return compactObject({ + id: pageGuid, + title, + createdAt: + created === undefined ? undefined : safeFiletime(context, created), + updatedAt: updated === undefined ? undefined : oneNoteTimeToIso(updated), + level, + text, + blocks, + }); +} + +function safeFiletime( + context: SectionContentContext, + value: bigint +): string | undefined { + try { + return filetimeToIso(value, 'PageMetadata creation time'); + } catch (error) { + addDiagnostic( + context, + 'INVALID_PAGE_TIMESTAMP', + error instanceof Error ? error.message : String(error), + 'PageMetadata' + ); + return undefined; + } +} + +function cleanMetadataString(value: string | undefined): string | undefined { + const result = value?.split('\0').join('').trim(); + return result || undefined; +} + +function cleanTitle(text: string): string | undefined { + const result = text + .split('\n') + .map((value) => value.trim()) + .find(Boolean); + return result || undefined; +} + +function firstNonemptyLine(text: string): string | undefined { + return text + .split('\n') + .map((value) => value.trim()) + .find(Boolean); +} + +function addDiagnostic( + context: SectionContentContext, + code: string, + message: string, + structure: string, + severity: 'warning' | 'error' = 'warning' +): void { + if (context.diagnostics.length >= context.limits.maxDiagnostics) return; + context.diagnostics.push({ + severity, + code, + message, + structure: context.currentPage + ? `${structure} (page ${context.currentPage})` + : structure, + recoverable: true, + }); +} + +function stripOneExtension(filename: string): string { + return (filename.split(/[\\/]/).at(-1) ?? filename).replace(/\.one$/i, ''); +} + +function compactObject(value: T): T { + for (const key of Object.keys(value) as (keyof T)[]) { + if (value[key] === undefined) delete value[key]; + } + return value; +} diff --git a/src/onenote/onestore/desktop-store.ts b/src/onenote/onestore/desktop-store.ts index 8de6e14..bad3677 100644 --- a/src/onenote/onestore/desktop-store.ts +++ b/src/onenote/onestore/desktop-store.ts @@ -8,7 +8,7 @@ // one_store_file.rs` and `onestore/desktop/objects/{global_id_table,object, // object_group_list,object_space,revision,revision_manifest_list, // root_file_node_list}.rs`. Deviations: selected section nodes only, -// serializable diagnostics, explicit limits, and attachment payloads skipped. +// serializable diagnostics, explicit limits, and zero-copy file-data ranges. import { BinaryReader } from '../binary/BinaryReader.js'; import { readGuid } from '../binary/guid.js'; @@ -26,6 +26,12 @@ import { readChunkReference64x32, readTransactionNodeCounts, } from './file-node.js'; +import { + emptyObjectPropSet, + parseFileDataDeclaration, + parseFileDataStore, + type FileDataStore, +} from './file-data.js'; import { parseObjectPropSet } from './property-set.js'; import type { CompactId, @@ -49,6 +55,7 @@ interface ParseContext { limits: OneNoteParserLimits; diagnostics: ParserDiagnostic[]; objectCount: number; + fileDataStore: FileDataStore; } export function parseDesktopOneStore( @@ -132,11 +139,13 @@ function parseDesktopRevisionStore( ); const fileNodeParser = new DesktopFileNodeParser(bytes, limits, nodeCounts); const rootList = fileNodeParser.parseList(rootReference); + const fileDataStore = parseFileDataStore(bytes, rootList, limits); const context: ParseContext = { bytes, limits, diagnostics, objectCount: 0, + fileDataStore, }; let rootObjectSpaceId: ExGuid | undefined; @@ -573,12 +582,40 @@ function parseObjectDeclaration( objects: Map ): void { if (node.id === 0x072 || node.id === 0x073) { - diagnostic( - context, - 'UNSUPPORTED_ATTACHMENT_DECLARATION', - 'Attachment object metadata is not extracted in this parser phase', - node + const declaration = parseFileDataDeclaration( + context.bytes, + node, + context.fileDataStore, + context.limits ); + const id = resolveCompactId(declaration.compactId, mapping); + objects.set(exGuidKey(id), { + id, + jcid: declaration.jcid, + props: emptyObjectPropSet(), + mapping: new Map(mapping), + contextId, + fileData: declaration.fileData, + }); + if ( + declaration.fileData.referenceKind === 'embedded' && + !declaration.fileData.blob + ) { + diagnostic( + context, + 'MISSING_FILE_DATA', + `File-data store does not contain ${declaration.fileData.storageGuid}`, + node + ); + } else if (declaration.fileData.referenceKind === 'external') { + diagnostic( + context, + 'EXTERNAL_FILE_DATA_UNAVAILABLE', + 'The file-data object refers to an external onefiles entry that was not selected', + node + ); + } + incrementObjectCount(context, node); return; } if (!node.dataReference) { @@ -665,6 +702,10 @@ function parseObjectDeclaration( mapping: new Map(mapping), contextId, }); + incrementObjectCount(context, node); +} + +function incrementObjectCount(context: ParseContext, node: FileNode): void { context.objectCount += 1; if (context.objectCount > context.limits.maxObjects) { throw new OneNoteParserError( diff --git a/src/onenote/onestore/file-data.ts b/src/onenote/onestore/file-data.ts new file mode 100644 index 0000000..77675fb --- /dev/null +++ b/src/onenote/onestore/file-data.ts @@ -0,0 +1,336 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 +// +// Portions adapted from onenote.rs (MPL-2.0), revision +// 5138a39a3f4e72b840932f9872fecde52fa9da60: `onestore/desktop/ +// {file_node/shared,objects/file_data_store}.rs`. Deviations: file payloads +// remain bounded zero-copy byte ranges, all string/count arithmetic is checked, +// and unavailable external/invalid references remain explicit metadata. + +import { BinaryReader } from '../binary/BinaryReader.js'; +import { normalizeGuid, readGuid } from '../binary/guid.js'; +import { OneNoteParserError } from '../parser/error.js'; +import type { OneNoteParserLimits } from '../parser/limits.js'; +import type { + CompactId, + ExGuid, + FileDataBlobReference, + FileNode, + FileNodeList, + ObjectPropSet, + StoreFileData, +} from './types.js'; + +const FILE_DATA_STORE_HEADER = '{BDE316E7-2665-4511-A4C4-8D4D0B7A9EAC}'; +const FILE_DATA_STORE_FOOTER = '{71FBA722-0F79-4A0B-BB13-899256426B24}'; +const FILE_DATA_FIXED_PREFIX_BYTES = 36; + +export type FileDataStore = ReadonlyMap; + +export interface ParsedFileDataDeclaration { + compactId: CompactId; + jcid: number; + fileData: StoreFileData; +} + +export function emptyObjectPropSet(): ObjectPropSet { + return { + objectIds: [], + objectSpaceIds: [], + contextIds: [], + properties: { entries: [] }, + }; +} + +/** Parse the optional root-level FileDataStoreListReferenceFND (0x090). */ +export function parseFileDataStore( + bytes: Uint8Array, + rootList: FileNodeList, + limits: OneNoteParserLimits +): FileDataStore { + const storeNodes = rootList.nodes.filter((node) => node.id === 0x090); + if (storeNodes.length > 1) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'Root FileNodeList contains more than one file-data store', + { structure: 'FileDataStoreListReferenceFND' } + ); + } + const storeNode = storeNodes[0]; + if (!storeNode) return new Map(); + if (!storeNode.childList) { + throw new OneNoteParserError( + 'MISSING_REFERENCE', + 'FileDataStoreListReferenceFND has no child FileNodeList', + { offset: storeNode.offset, structure: 'FileDataStoreListReferenceFND' } + ); + } + + const result = new Map(); + let totalBytes = 0; + for (const node of storeNode.childList.nodes) { + if (node.id !== 0x094) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + `Unexpected FileNode 0x${node.id.toString(16)} in file-data store`, + { offset: node.offset, structure: 'FileDataStoreList' } + ); + } + if (result.size >= limits.maxFileDataObjects) { + throw new OneNoteParserError( + 'LIMIT_EXCEEDED', + `File-data object count exceeds ${limits.maxFileDataObjects}`, + { offset: node.offset, structure: 'FileDataStoreList' } + ); + } + if ( + !node.dataReference || + node.dataReference.nil || + node.dataReference.zero + ) { + throw new OneNoteParserError( + 'MISSING_REFERENCE', + 'FileDataStoreObjectReferenceFND has no data reference', + { offset: node.offset, structure: 'FileDataStoreObjectReferenceFND' } + ); + } + + const nodeReader = new BinaryReader( + bytes, + node.payloadOffset, + node.payloadLength, + 'FileDataStoreObjectReferenceFND' + ); + const guid = readGuid(nodeReader); + if (nodeReader.remaining !== 0) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'FileDataStoreObjectReferenceFND has trailing payload bytes', + { + offset: nodeReader.offset, + structure: 'FileDataStoreObjectReferenceFND', + } + ); + } + if (result.has(guid)) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + `Duplicate file-data GUID ${guid}`, + { offset: node.offset, structure: 'FileDataStoreList' } + ); + } + + const blob = parseFileDataStoreObject(bytes, node, limits); + totalBytes = checkedAdd(totalBytes, blob.size, node.offset); + if (totalBytes > limits.maxTotalFileDataBytes) { + throw new OneNoteParserError( + 'LIMIT_EXCEEDED', + `File-data payloads exceed ${limits.maxTotalFileDataBytes} bytes`, + { offset: node.offset, structure: 'FileDataStoreList' } + ); + } + result.set(guid, blob); + } + return result; +} + +/** Parse ObjectDeclarationFileData3RefCountFND (0x072/0x073). */ +export function parseFileDataDeclaration( + bytes: Uint8Array, + node: FileNode, + store: FileDataStore, + limits: OneNoteParserLimits +): ParsedFileDataDeclaration { + if (node.id !== 0x072 && node.id !== 0x073) { + throw new TypeError('Expected a file-data object declaration'); + } + const reader = new BinaryReader( + bytes, + node.payloadOffset, + node.payloadLength, + 'ObjectDeclarationFileData3RefCountFND' + ); + const compactId = readCompactId(reader); + const jcid = reader.u32(); + if (node.id === 0x072) reader.u8(); + else reader.u32(); + const dataReference = readStorageString(reader, limits); + const extension = readStorageString(reader, limits); + if (reader.remaining !== 0) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'File-data declaration has trailing payload bytes', + { + offset: reader.offset, + structure: 'ObjectDeclarationFileData3RefCountFND', + } + ); + } + + return { + compactId, + jcid, + fileData: resolveFileData(dataReference, extension, store, node), + }; +} + +export function resolveCompactFileDataId( + compactId: CompactId, + mapping: ReadonlyMap +): ExGuid { + const guid = mapping.get(compactId.guidIndex); + if (!guid) { + throw new OneNoteParserError( + 'MISSING_REFERENCE', + `GlobalIdTable has no GUID at index ${compactId.guidIndex}`, + { structure: 'CompactId' } + ); + } + return { guid, value: compactId.value }; +} + +function parseFileDataStoreObject( + bytes: Uint8Array, + node: FileNode, + limits: OneNoteParserLimits +): FileDataBlobReference { + const reference = node.dataReference!; + const reader = new BinaryReader( + bytes, + reference.offset, + reference.size, + 'FileDataStoreObject' + ); + if (readGuid(reader) !== FILE_DATA_STORE_HEADER) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'Invalid FileDataStoreObject header GUID', + { offset: reference.offset, structure: 'FileDataStoreObject' } + ); + } + const rawLength = reader.u64(); + if (rawLength > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new OneNoteParserError( + 'LIMIT_EXCEEDED', + 'File-data length exceeds the safe integer range', + { offset: reader.offset - 8, structure: 'FileDataStoreObject' } + ); + } + const size = Number(rawLength); + if (size > limits.maxFileDataBytes) { + throw new OneNoteParserError( + 'LIMIT_EXCEEDED', + `File-data payload is ${size} bytes (limit ${limits.maxFileDataBytes})`, + { offset: reader.offset - 8, structure: 'FileDataStoreObject' } + ); + } + reader.u32(); // unused; consumers MUST ignore it + reader.u64(); // reserved; consumers MUST ignore it + const dataOffset = reader.offset; + reader.skip(size); + const padding = (8 - ((FILE_DATA_FIXED_PREFIX_BYTES + size) % 8)) % 8; + reader.skip(padding); + if (readGuid(reader) !== FILE_DATA_STORE_FOOTER) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'Invalid FileDataStoreObject footer GUID', + { offset: reader.offset - 16, structure: 'FileDataStoreObject' } + ); + } + if (reader.remaining !== 0) { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'FileDataStoreObject has trailing bytes', + { offset: reader.offset, structure: 'FileDataStoreObject' } + ); + } + return { source: bytes, offset: dataOffset, size }; +} + +function readStorageString( + reader: BinaryReader, + limits: OneNoteParserLimits +): string { + const characterCount = reader.u32(); + if (characterCount > Math.floor(limits.maxVectorBytes / 2)) { + throw new OneNoteParserError( + 'LIMIT_EXCEEDED', + `Storage string contains ${characterCount} UTF-16 code units`, + { offset: reader.offset - 4, structure: 'StringInStorageBuffer' } + ); + } + const byteLength = characterCount * 2; + try { + return new TextDecoder('utf-16le', { fatal: true }).decode( + reader.read(byteLength) + ); + } catch (error) { + if (error instanceof OneNoteParserError) throw error; + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + 'StringInStorageBuffer contains invalid UTF-16', + { offset: reader.offset - byteLength, structure: 'StringInStorageBuffer' } + ); + } +} + +function resolveFileData( + dataReference: string, + extension: string, + store: FileDataStore, + node: FileNode +): StoreFileData { + if (dataReference.startsWith('')) { + const rawGuid = dataReference.slice(''.length); + let storageGuid: string; + try { + storageGuid = normalizeGuid(rawGuid); + } catch { + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + `Invalid embedded file-data GUID ${rawGuid}`, + { + offset: node.offset, + structure: 'ObjectDeclarationFileData3RefCountFND', + } + ); + } + return { + referenceKind: 'embedded', + dataReference, + extension, + storageGuid, + blob: store.get(storageGuid), + }; + } + if (dataReference.startsWith('')) { + return { referenceKind: 'external', dataReference, extension }; + } + if (dataReference === '') { + return { referenceKind: 'invalid', dataReference, extension }; + } + throw new OneNoteParserError( + 'INVALID_STRUCTURE', + `Unknown file-data reference ${dataReference}`, + { offset: node.offset, structure: 'ObjectDeclarationFileData3RefCountFND' } + ); +} + +function readCompactId(reader: BinaryReader): CompactId { + const raw = reader.u32(); + return { value: raw & 0xff, guidIndex: raw >>> 8 }; +} + +function checkedAdd(left: number, right: number, offset: number): number { + const result = left + right; + if (!Number.isSafeInteger(result)) { + throw new OneNoteParserError( + 'LIMIT_EXCEEDED', + 'File-data byte count exceeds the safe integer range', + { offset, structure: 'FileDataStoreList' } + ); + } + return result; +} diff --git a/src/onenote/onestore/types.ts b/src/onenote/onestore/types.ts index 0c1186f..77415a9 100644 --- a/src/onenote/onestore/types.ts +++ b/src/onenote/onestore/types.ts @@ -76,12 +76,29 @@ export interface ObjectPropSet { properties: PropertySet; } +/** A bounded, zero-copy view of file data retained inside the source file. */ +export interface FileDataBlobReference { + source: Uint8Array; + offset: number; + size: number; +} + +/** Metadata carried by an ObjectDeclarationFileData3* FileNode. */ +export interface StoreFileData { + referenceKind: 'embedded' | 'external' | 'invalid'; + dataReference: string; + extension: string; + storageGuid?: string; + blob?: FileDataBlobReference; +} + export interface StoreObject { id: ExGuid; jcid: number; props: ObjectPropSet; mapping: ReadonlyMap; contextId: ExGuid; + fileData?: StoreFileData; } export interface ObjectSpace { @@ -93,3 +110,11 @@ export interface ObjectSpace { export function exGuidKey(id: ExGuid): string { return `${id.guid}:${id.value}`; } + +/** Return a view into the retained source without copying its bytes. */ +export function fileDataBytes(reference: FileDataBlobReference): Uint8Array { + return reference.source.subarray( + reference.offset, + reference.offset + reference.size + ); +} diff --git a/src/onenote/parser/limits.ts b/src/onenote/parser/limits.ts index 68a3b7b..9a30be9 100644 --- a/src/onenote/parser/limits.ts +++ b/src/onenote/parser/limits.ts @@ -15,7 +15,14 @@ export interface OneNoteParserLimits { maxStreamIds: number; maxVectorBytes: number; maxObjectReferences: number; + maxFileDataObjects: number; + maxFileDataBytes: number; + maxTotalFileDataBytes: number; maxGraphDepth: number; + maxContentBlocks: number; + maxTextRuns: number; + maxTableCells: number; + maxInkPoints: number; maxPages: number; maxDiagnostics: number; maxExtractedTextChars: number; @@ -34,7 +41,14 @@ export const DEFAULT_ONENOTE_PARSER_LIMITS: Readonly = { maxStreamIds: 1_000_000, maxVectorBytes: 64 * 1024 * 1024, maxObjectReferences: 1_000_000, + maxFileDataObjects: 10_000, + maxFileDataBytes: 256 * 1024 * 1024, + maxTotalFileDataBytes: 512 * 1024 * 1024, maxGraphDepth: 256, + maxContentBlocks: 1_000_000, + maxTextRuns: 1_000_000, + maxTableCells: 1_000_000, + maxInkPoints: 5_000_000, maxPages: 100_000, maxDiagnostics: 10_000, maxExtractedTextChars: 2_000_000, diff --git a/src/onenote/parser/parse-section.ts b/src/onenote/parser/parse-section.ts index 9c680dc..14258d5 100644 --- a/src/onenote/parser/parse-section.ts +++ b/src/onenote/parser/parse-section.ts @@ -3,7 +3,9 @@ // You can obtain one at https://mozilla.org/MPL/2.0/. import type { OneNoteSectionDto } from '../model/dto.js'; +import type { OneNoteSectionContentModel } from '../one/content-model.js'; import { buildSectionDto } from '../one/section.js'; +import { buildSectionContentModel } from '../one/section-content.js'; import { parseDesktopOneStore } from '../onestore/desktop-store.js'; import { toBytes } from './detect-format.js'; import type { OneNoteParserLimits } from './limits.js'; @@ -30,3 +32,20 @@ export function parseOneNoteSection( diagnostics ); } + +/** Parse a section into the internal rich-content and lazy-resource model. */ +export function parseOneNoteSectionContent( + input: ArrayBuffer | Uint8Array, + options: ParseOneNoteSectionOptions = {} +): OneNoteSectionContentModel { + const bytes = toBytes(input); + const limits = resolveParserLimits(options.limits); + const diagnostics: OneNoteSectionContentModel['diagnostics'] = []; + const store = parseDesktopOneStore(bytes, limits, diagnostics); + return buildSectionContentModel( + store, + options.sourceName ?? 'Untitled.one', + limits, + diagnostics + ); +} diff --git a/tests/content-model.test.ts b/tests/content-model.test.ts new file mode 100644 index 0000000..7d83f93 --- /dev/null +++ b/tests/content-model.test.ts @@ -0,0 +1,278 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { + DEFAULT_ONENOTE_PARSER_LIMITS, + parseOneNoteSectionContent, + type OneNoteContentBlock, + type OneNoteTextBlock, +} from '../src/onenote/index.js'; +import { parseContentObjects } from '../src/onenote/one/content.js'; +import { JCID, PROPERTY } from '../src/onenote/one/constants.js'; +import type { + ExGuid, + ObjectPropSet, + ObjectSpace, + PropertyEntry, + StoreObject, +} from '../src/onenote/onestore/types.js'; +import { exGuidKey } from '../src/onenote/onestore/types.js'; + +const fixture = Buffer.from( + readFileSync( + join(process.cwd(), 'tests/fixtures/onenote_desktop.one.b64'), + 'utf8' + ).replace(/\s/g, ''), + 'base64' +); + +describe('OneNote rich-content model', () => { + it('retains structured blocks, formatting, and lazy resources', () => { + const model = parseOneNoteSectionContent(fixture, { + sourceName: 'onenote_desktop.one', + }); + const blocks = model.pages.flatMap((page) => collectBlocks(page.blocks)); + + expect(model.pages).toHaveLength(3); + expect(model.pages[0]!.text).toContain('TEST\tD'); + expect(blocks.some((block) => block.kind === 'table')).toBe(true); + expect(blocks.some((block) => block.kind === 'image')).toBe(true); + + const testing = textBlock(blocks, 'Testing!'); + expect(testing.runs.map((run) => run.text)).toEqual(['Testing', '!']); + expect(testing.runs[0]!.style.underline).toBe(true); + expect(testing.runs[1]!.style.underline).toBe(false); + expect( + textBlock(blocks, 'This page is nearly empty').runs[1] + ).toMatchObject({ + text: 'nearly empty', + style: { italic: true }, + }); + + const table = blocks.find((block) => block.kind === 'table'); + if (!table || table.kind !== 'table') throw new Error('Table is missing'); + expect(table.rows.length).toBeGreaterThan(0); + expect(table.rows.flatMap((row) => row.cells).length).toBeGreaterThan(0); + expect(table.columnWidths.every(Number.isFinite)).toBe(true); + expect(table.lockedColumns).toHaveLength( + table.declaredColumnCount ?? table.lockedColumns.length + ); + + expect([...model.resources.values()]).toHaveLength(1); + const resource = [...model.resources.values()][0]!; + expect(resource).toMatchObject({ + kind: 'image', + extension: 'png', + mediaType: 'image/png', + browserRenderable: true, + availability: 'available', + size: 2639, + }); + expect(resource.blob?.source).toHaveLength(fixture.length); + expect( + resource.blob?.source.subarray( + resource.blob.offset, + resource.blob.offset + 8 + ) + ).toEqual(Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 13, 10, 26, 10)); + + const diagnosticCodes = model.diagnostics.map(({ code }) => code); + expect(diagnosticCodes).not.toContain('CONTENT_PARSE_FAILED'); + expect(diagnosticCodes).not.toContain('INVALID_TABLE_VECTOR'); + }); + + it('associates only allow-listed hyperlink marker targets', () => { + const safe = parseSyntheticLink('https://example.com/path'); + expect(safe.text).toBe('Example'); + expect(safe.runs.find((run) => run.text === 'Example')?.href).toBe( + 'https://example.com/path' + ); + + const unsafe = parseSyntheticLink('javascript:alert(1)'); + expect(unsafe.text).toBe('Example'); + expect( + unsafe.runs.find((run) => run.text === 'Example')?.href + ).toBeUndefined(); + }); +}); + +function parseSyntheticLink(href: string): OneNoteTextBlock { + const guid = '{11111111-1111-1111-1111-111111111111}'; + const richId = id(guid, 1); + const paragraphId = id(guid, 2); + const hiddenId = id(guid, 3); + const visibleId = id(guid, 4); + const marker = `\ufddfHYPERLINK "${href}"`; + const text = `${marker}Example`; + const objects = [ + object( + richId, + JCID.RichTextNode, + { + ...emptyProps(), + objectIds: [compact(2), compact(3), compact(4)], + properties: { + entries: [ + referenceEntry(PROPERTY.ParagraphStyle, 'one'), + referenceEntry(PROPERTY.TextRunFormatting, 'many', 2), + bytesEntry(PROPERTY.TextRunIndex, u32Vector(marker.length)), + bytesEntry( + PROPERTY.RichEditTextUnicode, + Buffer.from(text, 'utf16le') + ), + ], + }, + }, + guid + ), + object(paragraphId, JCID.ParagraphStyleObject, emptyProps(), guid), + object( + hiddenId, + JCID.ParagraphStyleObject, + props( + boolEntry(PROPERTY.Hidden, true), + boolEntry(PROPERTY.Hyperlink, true) + ), + guid + ), + object( + visibleId, + JCID.ParagraphStyleObject, + props( + boolEntry(PROPERTY.HyperlinkProtected, true), + boolEntry(PROPERTY.Underline, true) + ), + guid + ), + ]; + const space: ObjectSpace = { + id: id(guid, 0), + roots: new Map(), + objects: new Map(objects.map((value) => [exGuidKey(value.id), value])), + }; + const blocks = parseContentObjects([richId], { + space, + limits: { ...DEFAULT_ONENOTE_PARSER_LIMITS }, + diagnostics: [], + resources: new Map(), + }); + expect(blocks[0]?.kind).toBe('text'); + return blocks[0] as OneNoteTextBlock; +} + +function collectBlocks( + blocks: readonly OneNoteContentBlock[] +): OneNoteContentBlock[] { + const result: OneNoteContentBlock[] = []; + for (const block of blocks) { + result.push(block); + if (block.kind === 'outline' || block.kind === 'outline-group') { + result.push(...collectBlocks(block.children)); + } else if (block.kind === 'outline-element') { + result.push(...collectBlocks(block.contents)); + result.push(...collectBlocks(block.children)); + } else if (block.kind === 'table') { + for (const row of block.rows) { + for (const cell of row.cells) { + result.push(...collectBlocks(cell.blocks)); + } + } + } + } + return result; +} + +function textBlock( + blocks: readonly OneNoteContentBlock[], + text: string +): OneNoteTextBlock { + const result = blocks.find( + (block): block is OneNoteTextBlock => + block.kind === 'text' && block.text.includes(text) + ); + expect(result).toBeDefined(); + return result!; +} + +function object( + objectId: ExGuid, + jcid: number, + objectProps: ObjectPropSet, + mappedGuid: string +): StoreObject { + return { + id: objectId, + jcid, + props: objectProps, + mapping: new Map([[1, mappedGuid]]), + contextId: id(mappedGuid, 0), + }; +} + +function id(guid: string, value: number): ExGuid { + return { guid, value }; +} + +function compact(value: number): { guidIndex: number; value: number } { + return { guidIndex: 1, value }; +} + +function emptyProps(): ObjectPropSet { + return { + objectIds: [], + objectSpaceIds: [], + contextIds: [], + properties: { entries: [] }, + }; +} + +function props(...entries: PropertyEntry[]): ObjectPropSet { + return { ...emptyProps(), properties: { entries } }; +} + +function referenceEntry( + rawId: number, + cardinality: 'one' | 'many', + count = 1 +): PropertyEntry { + return { + rawId, + id: rawId & 0x03ffffff, + type: cardinality === 'one' ? 0x8 : 0x9, + value: + cardinality === 'one' + ? { kind: 'objectId' } + : { kind: 'objectIds', count }, + }; +} + +function bytesEntry(rawId: number, value: Uint8Array): PropertyEntry { + return { + rawId, + id: rawId & 0x03ffffff, + type: 0x7, + value: { kind: 'bytes', value }, + }; +} + +function boolEntry(rawId: number, value: boolean): PropertyEntry { + return { + rawId, + id: rawId & 0x03ffffff, + type: 0x2, + value: { kind: 'bool', value }, + }; +} + +function u32Vector(...values: number[]): Uint8Array { + const result = Buffer.alloc(values.length * 4); + values.forEach((value, index) => result.writeUInt32LE(value, index * 4)); + return result; +} diff --git a/tests/file-data.test.ts b/tests/file-data.test.ts new file mode 100644 index 0000000..088adb3 --- /dev/null +++ b/tests/file-data.test.ts @@ -0,0 +1,177 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 + +import { describe, expect, it } from 'vitest'; + +import { + parseFileDataDeclaration, + parseFileDataStore, +} from '../src/onenote/onestore/file-data.js'; +import type { FileNode, FileNodeList } from '../src/onenote/onestore/types.js'; +import { fileDataBytes } from '../src/onenote/onestore/types.js'; +import { resolveParserLimits } from '../src/onenote/parser/limits.js'; + +const HEADER = '{BDE316E7-2665-4511-A4C4-8D4D0B7A9EAC}'; +const FOOTER = '{71FBA722-0F79-4A0B-BB13-899256426B24}'; +const STORAGE_GUID = '{01234567-89AB-CDEF-8123-456789ABCDEF}'; + +describe('OneStore file-data objects', () => { + it('indexes bounded payload ranges and resolves 0x072 declarations', () => { + const payload = Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 13, 10, 26, 10); + const fixture = fileDataFixture(payload); + const store = parseFileDataStore( + fixture.bytes, + fixture.root, + resolveParserLimits() + ); + const blob = store.get(STORAGE_GUID)!; + + expect(blob).toMatchObject({ offset: 100, size: payload.length }); + expect(blob.source.buffer).toBe(fixture.bytes.buffer); + expect([...fileDataBytes(blob)]).toEqual([...payload]); + + const declarationBytes = declarationPayload( + 0x072, + `${STORAGE_GUID}`, + '.png' + ); + const declaration = parseFileDataDeclaration( + declarationBytes, + node(0x072, 0, declarationBytes.length), + store, + resolveParserLimits() + ); + expect(declaration).toMatchObject({ + compactId: { guidIndex: 3, value: 7 }, + jcid: 0x00080039, + fileData: { + referenceKind: 'embedded', + storageGuid: STORAGE_GUID, + extension: '.png', + blob, + }, + }); + }); + + it('keeps external references explicit and enforces payload limits', () => { + const externalBytes = declarationPayload( + 0x073, + 'media/image.onebin', + '.jpg' + ); + expect( + parseFileDataDeclaration( + externalBytes, + node(0x073, 0, externalBytes.length), + new Map(), + resolveParserLimits() + ).fileData + ).toEqual({ + referenceKind: 'external', + dataReference: 'media/image.onebin', + extension: '.jpg', + }); + + const fixture = fileDataFixture(Uint8Array.of(1, 2, 3, 4, 5)); + expect(() => + parseFileDataStore( + fixture.bytes, + fixture.root, + resolveParserLimits({ maxFileDataBytes: 4 }) + ) + ).toThrow(/payload is 5 bytes/); + }); +}); + +function fileDataFixture(payload: Uint8Array): { + bytes: Uint8Array; + root: FileNodeList; +} { + const objectOffset = 64; + const padding = (8 - ((36 + payload.length) % 8)) % 8; + const objectSize = 36 + payload.length + padding + 16; + const bytes = Buffer.alloc(objectOffset + objectSize); + writeGuid(bytes, 0, STORAGE_GUID); + writeGuid(bytes, objectOffset, HEADER); + bytes.writeBigUInt64LE(BigInt(payload.length), objectOffset + 16); + bytes.writeUInt32LE(0xfeedbeef, objectOffset + 24); + bytes.writeBigUInt64LE(0x1234n, objectOffset + 28); + bytes.set(payload, objectOffset + 36); + writeGuid(bytes, objectOffset + 36 + payload.length + padding, FOOTER); + + const dataNode: FileNode = { + ...node(0x094, 0, 16), + dataReference: { + offset: objectOffset, + size: objectSize, + nil: false, + zero: false, + }, + }; + return { + bytes, + root: { + id: 0, + nodes: [ + { + ...node(0x090, 0, 0), + childList: { id: 1, nodes: [dataNode] }, + }, + ], + }, + }; +} + +function declarationPayload( + id: 0x072 | 0x073, + dataReference: string, + extension: string +): Uint8Array { + const data = Buffer.from(dataReference, 'utf16le'); + const ext = Buffer.from(extension, 'utf16le'); + const countSize = id === 0x072 ? 1 : 4; + const result = Buffer.alloc( + 4 + 4 + countSize + 4 + data.length + 4 + ext.length + ); + let offset = 0; + result.writeUInt32LE((3 << 8) | 7, offset); + offset += 4; + result.writeUInt32LE(0x00080039, offset); + offset += 4; + if (id === 0x072) result.writeUInt8(1, offset); + else result.writeUInt32LE(1, offset); + offset += countSize; + result.writeUInt32LE(dataReference.length, offset); + offset += 4; + result.set(data, offset); + offset += data.length; + result.writeUInt32LE(extension.length, offset); + offset += 4; + result.set(ext, offset); + return result; +} + +function node( + id: number, + payloadOffset: number, + payloadLength: number +): FileNode { + return { + id, + declaredSize: payloadLength, + offset: payloadOffset, + payloadOffset, + payloadLength, + }; +} + +function writeGuid(target: Buffer, offset: number, guid: string): void { + const hex = guid.replace(/[{}-]/g, ''); + const bytes = Buffer.from(hex, 'hex'); + const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15]; + order.forEach((sourceIndex, index) => { + target[offset + index] = bytes[sourceIndex]!; + }); +}