feat: parse rich OneNote content resources
This commit is contained in:
278
tests/content-model.test.ts
Normal file
278
tests/content-model.test.ts
Normal file
@@ -0,0 +1,278 @@
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
DEFAULT_ONENOTE_PARSER_LIMITS,
|
||||
parseOneNoteSectionContent,
|
||||
type OneNoteContentBlock,
|
||||
type OneNoteTextBlock,
|
||||
} from '../src/onenote/index.js';
|
||||
import { parseContentObjects } from '../src/onenote/one/content.js';
|
||||
import { JCID, PROPERTY } from '../src/onenote/one/constants.js';
|
||||
import type {
|
||||
ExGuid,
|
||||
ObjectPropSet,
|
||||
ObjectSpace,
|
||||
PropertyEntry,
|
||||
StoreObject,
|
||||
} from '../src/onenote/onestore/types.js';
|
||||
import { exGuidKey } from '../src/onenote/onestore/types.js';
|
||||
|
||||
const fixture = Buffer.from(
|
||||
readFileSync(
|
||||
join(process.cwd(), 'tests/fixtures/onenote_desktop.one.b64'),
|
||||
'utf8'
|
||||
).replace(/\s/g, ''),
|
||||
'base64'
|
||||
);
|
||||
|
||||
describe('OneNote rich-content model', () => {
|
||||
it('retains structured blocks, formatting, and lazy resources', () => {
|
||||
const model = parseOneNoteSectionContent(fixture, {
|
||||
sourceName: 'onenote_desktop.one',
|
||||
});
|
||||
const blocks = model.pages.flatMap((page) => collectBlocks(page.blocks));
|
||||
|
||||
expect(model.pages).toHaveLength(3);
|
||||
expect(model.pages[0]!.text).toContain('TEST\tD');
|
||||
expect(blocks.some((block) => block.kind === 'table')).toBe(true);
|
||||
expect(blocks.some((block) => block.kind === 'image')).toBe(true);
|
||||
|
||||
const testing = textBlock(blocks, 'Testing!');
|
||||
expect(testing.runs.map((run) => run.text)).toEqual(['Testing', '!']);
|
||||
expect(testing.runs[0]!.style.underline).toBe(true);
|
||||
expect(testing.runs[1]!.style.underline).toBe(false);
|
||||
expect(
|
||||
textBlock(blocks, 'This page is nearly empty').runs[1]
|
||||
).toMatchObject({
|
||||
text: 'nearly empty',
|
||||
style: { italic: true },
|
||||
});
|
||||
|
||||
const table = blocks.find((block) => block.kind === 'table');
|
||||
if (!table || table.kind !== 'table') throw new Error('Table is missing');
|
||||
expect(table.rows.length).toBeGreaterThan(0);
|
||||
expect(table.rows.flatMap((row) => row.cells).length).toBeGreaterThan(0);
|
||||
expect(table.columnWidths.every(Number.isFinite)).toBe(true);
|
||||
expect(table.lockedColumns).toHaveLength(
|
||||
table.declaredColumnCount ?? table.lockedColumns.length
|
||||
);
|
||||
|
||||
expect([...model.resources.values()]).toHaveLength(1);
|
||||
const resource = [...model.resources.values()][0]!;
|
||||
expect(resource).toMatchObject({
|
||||
kind: 'image',
|
||||
extension: 'png',
|
||||
mediaType: 'image/png',
|
||||
browserRenderable: true,
|
||||
availability: 'available',
|
||||
size: 2639,
|
||||
});
|
||||
expect(resource.blob?.source).toHaveLength(fixture.length);
|
||||
expect(
|
||||
resource.blob?.source.subarray(
|
||||
resource.blob.offset,
|
||||
resource.blob.offset + 8
|
||||
)
|
||||
).toEqual(Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 13, 10, 26, 10));
|
||||
|
||||
const diagnosticCodes = model.diagnostics.map(({ code }) => code);
|
||||
expect(diagnosticCodes).not.toContain('CONTENT_PARSE_FAILED');
|
||||
expect(diagnosticCodes).not.toContain('INVALID_TABLE_VECTOR');
|
||||
});
|
||||
|
||||
it('associates only allow-listed hyperlink marker targets', () => {
|
||||
const safe = parseSyntheticLink('https://example.com/path');
|
||||
expect(safe.text).toBe('Example');
|
||||
expect(safe.runs.find((run) => run.text === 'Example')?.href).toBe(
|
||||
'https://example.com/path'
|
||||
);
|
||||
|
||||
const unsafe = parseSyntheticLink('javascript:alert(1)');
|
||||
expect(unsafe.text).toBe('Example');
|
||||
expect(
|
||||
unsafe.runs.find((run) => run.text === 'Example')?.href
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
function parseSyntheticLink(href: string): OneNoteTextBlock {
|
||||
const guid = '{11111111-1111-1111-1111-111111111111}';
|
||||
const richId = id(guid, 1);
|
||||
const paragraphId = id(guid, 2);
|
||||
const hiddenId = id(guid, 3);
|
||||
const visibleId = id(guid, 4);
|
||||
const marker = `\ufddfHYPERLINK "${href}"`;
|
||||
const text = `${marker}Example`;
|
||||
const objects = [
|
||||
object(
|
||||
richId,
|
||||
JCID.RichTextNode,
|
||||
{
|
||||
...emptyProps(),
|
||||
objectIds: [compact(2), compact(3), compact(4)],
|
||||
properties: {
|
||||
entries: [
|
||||
referenceEntry(PROPERTY.ParagraphStyle, 'one'),
|
||||
referenceEntry(PROPERTY.TextRunFormatting, 'many', 2),
|
||||
bytesEntry(PROPERTY.TextRunIndex, u32Vector(marker.length)),
|
||||
bytesEntry(
|
||||
PROPERTY.RichEditTextUnicode,
|
||||
Buffer.from(text, 'utf16le')
|
||||
),
|
||||
],
|
||||
},
|
||||
},
|
||||
guid
|
||||
),
|
||||
object(paragraphId, JCID.ParagraphStyleObject, emptyProps(), guid),
|
||||
object(
|
||||
hiddenId,
|
||||
JCID.ParagraphStyleObject,
|
||||
props(
|
||||
boolEntry(PROPERTY.Hidden, true),
|
||||
boolEntry(PROPERTY.Hyperlink, true)
|
||||
),
|
||||
guid
|
||||
),
|
||||
object(
|
||||
visibleId,
|
||||
JCID.ParagraphStyleObject,
|
||||
props(
|
||||
boolEntry(PROPERTY.HyperlinkProtected, true),
|
||||
boolEntry(PROPERTY.Underline, true)
|
||||
),
|
||||
guid
|
||||
),
|
||||
];
|
||||
const space: ObjectSpace = {
|
||||
id: id(guid, 0),
|
||||
roots: new Map(),
|
||||
objects: new Map(objects.map((value) => [exGuidKey(value.id), value])),
|
||||
};
|
||||
const blocks = parseContentObjects([richId], {
|
||||
space,
|
||||
limits: { ...DEFAULT_ONENOTE_PARSER_LIMITS },
|
||||
diagnostics: [],
|
||||
resources: new Map(),
|
||||
});
|
||||
expect(blocks[0]?.kind).toBe('text');
|
||||
return blocks[0] as OneNoteTextBlock;
|
||||
}
|
||||
|
||||
function collectBlocks(
|
||||
blocks: readonly OneNoteContentBlock[]
|
||||
): OneNoteContentBlock[] {
|
||||
const result: OneNoteContentBlock[] = [];
|
||||
for (const block of blocks) {
|
||||
result.push(block);
|
||||
if (block.kind === 'outline' || block.kind === 'outline-group') {
|
||||
result.push(...collectBlocks(block.children));
|
||||
} else if (block.kind === 'outline-element') {
|
||||
result.push(...collectBlocks(block.contents));
|
||||
result.push(...collectBlocks(block.children));
|
||||
} else if (block.kind === 'table') {
|
||||
for (const row of block.rows) {
|
||||
for (const cell of row.cells) {
|
||||
result.push(...collectBlocks(cell.blocks));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function textBlock(
|
||||
blocks: readonly OneNoteContentBlock[],
|
||||
text: string
|
||||
): OneNoteTextBlock {
|
||||
const result = blocks.find(
|
||||
(block): block is OneNoteTextBlock =>
|
||||
block.kind === 'text' && block.text.includes(text)
|
||||
);
|
||||
expect(result).toBeDefined();
|
||||
return result!;
|
||||
}
|
||||
|
||||
function object(
|
||||
objectId: ExGuid,
|
||||
jcid: number,
|
||||
objectProps: ObjectPropSet,
|
||||
mappedGuid: string
|
||||
): StoreObject {
|
||||
return {
|
||||
id: objectId,
|
||||
jcid,
|
||||
props: objectProps,
|
||||
mapping: new Map([[1, mappedGuid]]),
|
||||
contextId: id(mappedGuid, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function id(guid: string, value: number): ExGuid {
|
||||
return { guid, value };
|
||||
}
|
||||
|
||||
function compact(value: number): { guidIndex: number; value: number } {
|
||||
return { guidIndex: 1, value };
|
||||
}
|
||||
|
||||
function emptyProps(): ObjectPropSet {
|
||||
return {
|
||||
objectIds: [],
|
||||
objectSpaceIds: [],
|
||||
contextIds: [],
|
||||
properties: { entries: [] },
|
||||
};
|
||||
}
|
||||
|
||||
function props(...entries: PropertyEntry[]): ObjectPropSet {
|
||||
return { ...emptyProps(), properties: { entries } };
|
||||
}
|
||||
|
||||
function referenceEntry(
|
||||
rawId: number,
|
||||
cardinality: 'one' | 'many',
|
||||
count = 1
|
||||
): PropertyEntry {
|
||||
return {
|
||||
rawId,
|
||||
id: rawId & 0x03ffffff,
|
||||
type: cardinality === 'one' ? 0x8 : 0x9,
|
||||
value:
|
||||
cardinality === 'one'
|
||||
? { kind: 'objectId' }
|
||||
: { kind: 'objectIds', count },
|
||||
};
|
||||
}
|
||||
|
||||
function bytesEntry(rawId: number, value: Uint8Array): PropertyEntry {
|
||||
return {
|
||||
rawId,
|
||||
id: rawId & 0x03ffffff,
|
||||
type: 0x7,
|
||||
value: { kind: 'bytes', value },
|
||||
};
|
||||
}
|
||||
|
||||
function boolEntry(rawId: number, value: boolean): PropertyEntry {
|
||||
return {
|
||||
rawId,
|
||||
id: rawId & 0x03ffffff,
|
||||
type: 0x2,
|
||||
value: { kind: 'bool', value },
|
||||
};
|
||||
}
|
||||
|
||||
function u32Vector(...values: number[]): Uint8Array {
|
||||
const result = Buffer.alloc(values.length * 4);
|
||||
values.forEach((value, index) => result.writeUInt32LE(value, index * 4));
|
||||
return result;
|
||||
}
|
||||
177
tests/file-data.test.ts
Normal file
177
tests/file-data.test.ts
Normal file
@@ -0,0 +1,177 @@
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
parseFileDataDeclaration,
|
||||
parseFileDataStore,
|
||||
} from '../src/onenote/onestore/file-data.js';
|
||||
import type { FileNode, FileNodeList } from '../src/onenote/onestore/types.js';
|
||||
import { fileDataBytes } from '../src/onenote/onestore/types.js';
|
||||
import { resolveParserLimits } from '../src/onenote/parser/limits.js';
|
||||
|
||||
const HEADER = '{BDE316E7-2665-4511-A4C4-8D4D0B7A9EAC}';
|
||||
const FOOTER = '{71FBA722-0F79-4A0B-BB13-899256426B24}';
|
||||
const STORAGE_GUID = '{01234567-89AB-CDEF-8123-456789ABCDEF}';
|
||||
|
||||
describe('OneStore file-data objects', () => {
|
||||
it('indexes bounded payload ranges and resolves 0x072 declarations', () => {
|
||||
const payload = Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 13, 10, 26, 10);
|
||||
const fixture = fileDataFixture(payload);
|
||||
const store = parseFileDataStore(
|
||||
fixture.bytes,
|
||||
fixture.root,
|
||||
resolveParserLimits()
|
||||
);
|
||||
const blob = store.get(STORAGE_GUID)!;
|
||||
|
||||
expect(blob).toMatchObject({ offset: 100, size: payload.length });
|
||||
expect(blob.source.buffer).toBe(fixture.bytes.buffer);
|
||||
expect([...fileDataBytes(blob)]).toEqual([...payload]);
|
||||
|
||||
const declarationBytes = declarationPayload(
|
||||
0x072,
|
||||
`<ifndf>${STORAGE_GUID}`,
|
||||
'.png'
|
||||
);
|
||||
const declaration = parseFileDataDeclaration(
|
||||
declarationBytes,
|
||||
node(0x072, 0, declarationBytes.length),
|
||||
store,
|
||||
resolveParserLimits()
|
||||
);
|
||||
expect(declaration).toMatchObject({
|
||||
compactId: { guidIndex: 3, value: 7 },
|
||||
jcid: 0x00080039,
|
||||
fileData: {
|
||||
referenceKind: 'embedded',
|
||||
storageGuid: STORAGE_GUID,
|
||||
extension: '.png',
|
||||
blob,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps external references explicit and enforces payload limits', () => {
|
||||
const externalBytes = declarationPayload(
|
||||
0x073,
|
||||
'<file>media/image.onebin',
|
||||
'.jpg'
|
||||
);
|
||||
expect(
|
||||
parseFileDataDeclaration(
|
||||
externalBytes,
|
||||
node(0x073, 0, externalBytes.length),
|
||||
new Map(),
|
||||
resolveParserLimits()
|
||||
).fileData
|
||||
).toEqual({
|
||||
referenceKind: 'external',
|
||||
dataReference: '<file>media/image.onebin',
|
||||
extension: '.jpg',
|
||||
});
|
||||
|
||||
const fixture = fileDataFixture(Uint8Array.of(1, 2, 3, 4, 5));
|
||||
expect(() =>
|
||||
parseFileDataStore(
|
||||
fixture.bytes,
|
||||
fixture.root,
|
||||
resolveParserLimits({ maxFileDataBytes: 4 })
|
||||
)
|
||||
).toThrow(/payload is 5 bytes/);
|
||||
});
|
||||
});
|
||||
|
||||
function fileDataFixture(payload: Uint8Array): {
|
||||
bytes: Uint8Array;
|
||||
root: FileNodeList;
|
||||
} {
|
||||
const objectOffset = 64;
|
||||
const padding = (8 - ((36 + payload.length) % 8)) % 8;
|
||||
const objectSize = 36 + payload.length + padding + 16;
|
||||
const bytes = Buffer.alloc(objectOffset + objectSize);
|
||||
writeGuid(bytes, 0, STORAGE_GUID);
|
||||
writeGuid(bytes, objectOffset, HEADER);
|
||||
bytes.writeBigUInt64LE(BigInt(payload.length), objectOffset + 16);
|
||||
bytes.writeUInt32LE(0xfeedbeef, objectOffset + 24);
|
||||
bytes.writeBigUInt64LE(0x1234n, objectOffset + 28);
|
||||
bytes.set(payload, objectOffset + 36);
|
||||
writeGuid(bytes, objectOffset + 36 + payload.length + padding, FOOTER);
|
||||
|
||||
const dataNode: FileNode = {
|
||||
...node(0x094, 0, 16),
|
||||
dataReference: {
|
||||
offset: objectOffset,
|
||||
size: objectSize,
|
||||
nil: false,
|
||||
zero: false,
|
||||
},
|
||||
};
|
||||
return {
|
||||
bytes,
|
||||
root: {
|
||||
id: 0,
|
||||
nodes: [
|
||||
{
|
||||
...node(0x090, 0, 0),
|
||||
childList: { id: 1, nodes: [dataNode] },
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function declarationPayload(
|
||||
id: 0x072 | 0x073,
|
||||
dataReference: string,
|
||||
extension: string
|
||||
): Uint8Array {
|
||||
const data = Buffer.from(dataReference, 'utf16le');
|
||||
const ext = Buffer.from(extension, 'utf16le');
|
||||
const countSize = id === 0x072 ? 1 : 4;
|
||||
const result = Buffer.alloc(
|
||||
4 + 4 + countSize + 4 + data.length + 4 + ext.length
|
||||
);
|
||||
let offset = 0;
|
||||
result.writeUInt32LE((3 << 8) | 7, offset);
|
||||
offset += 4;
|
||||
result.writeUInt32LE(0x00080039, offset);
|
||||
offset += 4;
|
||||
if (id === 0x072) result.writeUInt8(1, offset);
|
||||
else result.writeUInt32LE(1, offset);
|
||||
offset += countSize;
|
||||
result.writeUInt32LE(dataReference.length, offset);
|
||||
offset += 4;
|
||||
result.set(data, offset);
|
||||
offset += data.length;
|
||||
result.writeUInt32LE(extension.length, offset);
|
||||
offset += 4;
|
||||
result.set(ext, offset);
|
||||
return result;
|
||||
}
|
||||
|
||||
function node(
|
||||
id: number,
|
||||
payloadOffset: number,
|
||||
payloadLength: number
|
||||
): FileNode {
|
||||
return {
|
||||
id,
|
||||
declaredSize: payloadLength,
|
||||
offset: payloadOffset,
|
||||
payloadOffset,
|
||||
payloadLength,
|
||||
};
|
||||
}
|
||||
|
||||
function writeGuid(target: Buffer, offset: number, guid: string): void {
|
||||
const hex = guid.replace(/[{}-]/g, '');
|
||||
const bytes = Buffer.from(hex, 'hex');
|
||||
const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15];
|
||||
order.forEach((sourceIndex, index) => {
|
||||
target[offset + index] = bytes[sourceIndex]!;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user