feat: decode OneNote ink strokes

src/onenote/index.ts

@@ -19,6 +19,10 @@ export type {
   OneNoteColor,
   OneNoteContentBlock,
   OneNoteImageBlock,
+  OneNoteInkBlock,
+  OneNoteInkBoundingBox,
+  OneNoteInkPoint,
+  OneNoteInkStroke,
   OneNotePageContentModel,
   OneNoteResource,
   OneNoteSectionContentModel,

src/onenote/one/constants.ts

@@ -112,6 +112,7 @@ export const PROPERTY = {
   InkScalingY: 0x14001c47,
   InkBoundingBox: 0x1c003418,
   InkPath: 0x1c00340b,
+  InkBias: 0x0c00341c,
   InkStrokeProperties: 0x20003409,
   InkDimensions: 0x1c00340a,
   InkPenTip: 0x0c003412,

src/onenote/one/content-model.ts

@@ -145,11 +145,37 @@ export interface OneNoteOutlineGroupBlock {
   children: OneNoteContentBlock[];
 }
 
+export interface OneNoteInkPoint {
+  x: number;
+  y: number;
+}
+
+export interface OneNoteInkBoundingBox {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface OneNoteInkStroke {
+  points: OneNoteInkPoint[];
+  penTip?: number;
+  transparency?: number;
+  width?: number;
+  height?: number;
+  color?: number;
+  bias?: 'handwriting' | 'drawing' | 'both';
+  languageCode?: number;
+}
+
+/** A leaf has strokes; an intermediate grouping container has children. */
 export interface OneNoteInkBlock {
   kind: 'ink';
   id: string;
-  supported: false;
+  strokes: OneNoteInkStroke[];
-  description: string;
+  children: OneNoteInkBlock[];
+  boundingBox?: OneNoteInkBoundingBox;
+  layout: OneNoteLayout;
 }
 
 export interface OneNoteUnsupportedBlock {

src/onenote/one/content.ts

@@ -10,6 +10,7 @@
 // allow-listed; file payloads remain zero-copy references.
 
 import { BinaryReader } from '../binary/BinaryReader.js';
+import { readGuid } from '../binary/guid.js';
 import type { ParserDiagnostic } from '../model/diagnostics.js';
 import type { OneNoteParserLimits } from '../parser/limits.js';
 import type {
@@ -24,6 +25,9 @@ import type {
   OneNoteColor,
   OneNoteContentBlock,
   OneNoteImageBlock,
+  OneNoteInkBlock,
+  OneNoteInkBoundingBox,
+  OneNoteInkStroke,
   OneNoteLayout,
   OneNoteResource,
   OneNoteTableBlock,
@@ -51,6 +55,7 @@ export interface OneNoteContentBudget {
   blocks: number;
   textRuns: number;
   tableCells: number;
+  inkPoints: number;
 }
 
 export interface OneNoteContentParseOptions {
@@ -67,7 +72,7 @@ interface ContentContext extends OneNoteContentParseOptions {
 }
 
 export function createContentBudget(): OneNoteContentBudget {
-  return { blocks: 0, textRuns: 0, tableCells: 0 };
+  return { blocks: 0, textRuns: 0, tableCells: 0, inkPoints: 0 };
 }
 
 /** Parse page graph roots into an inert, recursively structured content tree. */
@@ -189,18 +194,7 @@ function parseObject(
       case JCID.TableNode:
         return parseTable(context, object, active, depth);
       case JCID.InkContainer:
-        addDiagnostic(
+        return parseInk(context, object, active, depth);
-          context,
-          'UNSUPPORTED_INK',
-          'Ink content is retained as a typed placeholder',
-          `Object ${key}`
-        );
-        return {
-          kind: 'ink',
-          id: key,
-          supported: false,
-          description: 'Ink parsing is not implemented in this slice',
-        };
       default:
         addDiagnostic(
           context,
@@ -591,6 +585,342 @@ function parseTable(
   };
 }
 
+function parseInk(
+  context: ContentContext,
+  object: StoreObject,
+  active: Set<string>,
+  depth: number
+): OneNoteInkBlock {
+  const inkDataId = objectReferences(object, PROPERTY.InkData)[0];
+  if (!inkDataId) {
+    const children: OneNoteInkBlock[] = [];
+    for (const childId of objectReferences(
+      object,
+      PROPERTY.ContentChildNodes
+    )) {
+      const child = parseSafely(context, childId, active, depth + 1);
+      if (child?.kind === 'ink') {
+        children.push(child);
+      } else if (child) {
+        addDiagnostic(
+          context,
+          'INVALID_INK_CHILD',
+          `Ink group child ${exGuidKey(childId)} is not an ink container`,
+          `InkContainer ${exGuidKey(object.id)}`
+        );
+      }
+    }
+    return compactObject({
+      kind: 'ink' as const,
+      id: exGuidKey(object.id),
+      strokes: [],
+      children,
+      boundingBox: unionBoundingBoxes(
+        children.map((child) => child.boundingBox)
+      ),
+      layout: parseLayout(object),
+    });
+  }
+
+  const inkData = context.space.objects.get(exGuidKey(inkDataId));
+  if (!inkData) {
+    addDiagnostic(
+      context,
+      'MISSING_INK_DATA',
+      `Ink data ${exGuidKey(inkDataId)} is missing`,
+      `InkContainer ${exGuidKey(object.id)}`
+    );
+    return {
+      kind: 'ink',
+      id: exGuidKey(object.id),
+      strokes: [],
+      children: [],
+      layout: parseLayout(object),
+    };
+  }
+  assertJcid(inkData, JCID.InkDataNode, 'InkDataNode');
+  const scaleX = optionalF32(object, PROPERTY.InkScalingX) ?? 1;
+  const scaleY = optionalF32(object, PROPERTY.InkScalingY) ?? 1;
+  const strokes: OneNoteInkStroke[] = [];
+  for (const strokeId of objectReferences(inkData, PROPERTY.InkStrokes)) {
+    const stroke = parseInkStrokeSafely(context, strokeId, scaleX, scaleY);
+    if (stroke) strokes.push(stroke);
+  }
+  const encodedBoundingBox = parseInkBoundingBox(
+    optionalBytes(inkData, PROPERTY.InkBoundingBox),
+    scaleX,
+    scaleY
+  );
+  return compactObject({
+    kind: 'ink' as const,
+    id: exGuidKey(object.id),
+    strokes,
+    children: [],
+    boundingBox: boundingBoxFromStrokes(strokes) ?? encodedBoundingBox,
+    layout: parseLayout(object),
+  });
+}
+
+function parseInkStrokeSafely(
+  context: ContentContext,
+  id: ExGuid,
+  scaleX: number,
+  scaleY: number
+): OneNoteInkStroke | undefined {
+  try {
+    return parseInkStroke(context, id, scaleX, scaleY);
+  } catch (error) {
+    addDiagnostic(
+      context,
+      'INK_STROKE_PARSE_FAILED',
+      `Could not parse ink stroke ${exGuidKey(id)}: ${error instanceof Error ? error.message : String(error)}`,
+      'InkStrokeNode'
+    );
+    return undefined;
+  }
+}
+
+function parseInkStroke(
+  context: ContentContext,
+  id: ExGuid,
+  scaleX: number,
+  scaleY: number
+): OneNoteInkStroke {
+  const object = getObject(context.space, id, 'ink stroke');
+  assertJcid(object, JCID.InkStrokeNode, 'InkStrokeNode');
+  const propertiesId = objectReferences(
+    object,
+    PROPERTY.InkStrokeProperties
+  )[0];
+  if (!propertiesId) throw new Error('InkStrokeNode has no properties');
+  const properties = getObject(
+    context.space,
+    propertiesId,
+    'ink stroke properties'
+  );
+  assertJcid(properties, JCID.StrokePropertiesNode, 'StrokePropertiesNode');
+  const dimensions = parseInkDimensions(
+    optionalBytes(properties, PROPERTY.InkDimensions)
+  );
+  if (dimensions.length === 0 || dimensions.length > 64) {
+    throw new Error(`Invalid ink dimension count ${dimensions.length}`);
+  }
+  const xIndex = dimensions.findIndex(
+    (dimension) => dimension.id === INK_X_DIMENSION
+  );
+  const yIndex = dimensions.findIndex(
+    (dimension) => dimension.id === INK_Y_DIMENSION
+  );
+  if (xIndex < 0 || yIndex < 0) {
+    throw new Error('Ink stroke dimensions do not contain X and Y');
+  }
+  const pathBytes = optionalBytes(object, PROPERTY.InkPath);
+  if (!pathBytes) throw new Error('InkStrokeNode has no path');
+  const values = decodeSignedMultiByte(
+    pathBytes,
+    context.limits.maxInkPoints * dimensions.length
+  );
+  if (values.length % dimensions.length !== 0) {
+    throw new Error(
+      `Ink path has ${values.length} values for ${dimensions.length} dimensions`
+    );
+  }
+  const pointCount = values.length / dimensions.length;
+  incrementInkPointBudget(context, pointCount);
+  const xOffset = pointCount * xIndex;
+  const yOffset = pointCount * yIndex;
+  const xCoordinates = restoreInkDeltas(
+    values.slice(xOffset, xOffset + pointCount)
+  );
+  const yCoordinates = restoreInkDeltas(
+    values.slice(yOffset, yOffset + pointCount)
+  );
+  const points = Array.from({ length: pointCount }, (_, index) => ({
+    x: finiteInkCoordinate(xCoordinates[index]! * scaleX),
+    y: finiteInkCoordinate(yCoordinates[index]! * scaleY),
+  }));
+  return compactObject({
+    points,
+    penTip: optionalU8(properties, PROPERTY.InkPenTip),
+    transparency: optionalU8(properties, PROPERTY.InkTransparency),
+    // These two undocumented property names are reversed in observed files.
+    width: optionalF32(properties, PROPERTY.InkHeight),
+    height: optionalF32(properties, PROPERTY.InkWidth),
+    color: optionalU32(properties, PROPERTY.InkColor),
+    bias: inkBias(optionalU8(object, PROPERTY.InkBias)),
+    languageCode: optionalU32(object, PROPERTY.LanguageId),
+  });
+}
+
+interface InkDimension {
+  id: string;
+  lower: number;
+  upper: number;
+}
+
+const INK_X_DIMENSION = '{598A6A8F-52C0-4BA0-93AF-AF357411A561}';
+const INK_Y_DIMENSION = '{B53F9F75-04E0-4498-A7EE-C30DBB5A9011}';
+
+function parseInkDimensions(bytes: Uint8Array | undefined): InkDimension[] {
+  if (!bytes) return [];
+  if (bytes.length % 32 !== 0) {
+    throw new Error(
+      `InkDimensions length ${bytes.length} is not a multiple of 32`
+    );
+  }
+  const result: InkDimension[] = [];
+  const reader = new BinaryReader(bytes, 0, bytes.length, 'InkDimensions');
+  while (reader.remaining > 0) {
+    result.push({
+      id: readGuid(reader),
+      lower: signedU32(reader.u32()),
+      upper: signedU32(reader.u32()),
+    });
+    reader.skip(8);
+  }
+  return result;
+}
+
+function parseInkBoundingBox(
+  bytes: Uint8Array | undefined,
+  scaleX: number,
+  scaleY: number
+): OneNoteInkBoundingBox | undefined {
+  if (!bytes || bytes.length !== 16) return undefined;
+  const reader = new BinaryReader(bytes, 0, 16, 'InkBoundingBox');
+  const xMin = signedU32(reader.u32());
+  const yMin = signedU32(reader.u32());
+  const xMax = signedU32(reader.u32());
+  const yMax = signedU32(reader.u32());
+  return {
+    x: finiteInkCoordinate(xMin * scaleX),
+    y: finiteInkCoordinate(yMin * scaleY),
+    width: finiteInkCoordinate((xMax - xMin) * scaleX),
+    height: finiteInkCoordinate((yMax - yMin) * scaleY),
+  };
+}
+
+function decodeSignedMultiByte(bytes: Uint8Array, limit: number): number[] {
+  const lengthValue = decodeMultiByteInteger(bytes, 0);
+  const countValue = lengthValue.value >> 1n;
+  if (
+    countValue > BigInt(limit) ||
+    countValue > BigInt(Number.MAX_SAFE_INTEGER)
+  ) {
+    throw new Error(`Ink path value count exceeds ${limit}`);
+  }
+  const count = Number(countValue);
+  const result: number[] = [];
+  let offset = lengthValue.nextOffset;
+  for (let index = 0; index < count; index += 1) {
+    const decoded = decodeMultiByteInteger(bytes, offset);
+    offset = decoded.nextOffset;
+    const shifted = decoded.value >> 1n;
+    const signed = (decoded.value & 1n) === 1n ? -shifted : shifted;
+    if (
+      signed < BigInt(Number.MIN_SAFE_INTEGER) ||
+      signed > BigInt(Number.MAX_SAFE_INTEGER)
+    ) {
+      throw new Error('Ink coordinate exceeds the safe integer range');
+    }
+    result.push(Number(signed));
+  }
+  if (offset !== bytes.length) {
+    throw new Error(`Ink path has ${bytes.length - offset} trailing bytes`);
+  }
+  return result;
+}
+
+function decodeMultiByteInteger(
+  bytes: Uint8Array,
+  start: number
+): { value: bigint; nextOffset: number } {
+  let value = 0n;
+  let offset = start;
+  for (let count = 0; count < 10; count += 1) {
+    if (offset >= bytes.length)
+      throw new Error('Truncated ink multi-byte integer');
+    const byte = bytes[offset++]!;
+    value |= BigInt(byte & 0x7f) << BigInt(count * 7);
+    if ((byte & 0x80) === 0) return { value, nextOffset: offset };
+  }
+  throw new Error('Ink multi-byte integer exceeds 64-bit width');
+}
+
+function signedU32(value: number): number {
+  return value > 0x7fffffff ? value - 0x1_0000_0000 : value;
+}
+
+/** OneNote stores the first coordinate followed by signed coordinate deltas. */
+function restoreInkDeltas(values: readonly number[]): number[] {
+  if (values.length === 0) return [];
+  const result = [values[0]!];
+  let coordinate = values[0]!;
+  for (let index = 1; index < values.length; index += 1) {
+    coordinate = checkedInkInteger(coordinate + values[index]!);
+    result.push(coordinate);
+  }
+  return result;
+}
+
+function checkedInkInteger(value: number): number {
+  if (!Number.isSafeInteger(value)) {
+    throw new Error(
+      'Reconstructed ink coordinate exceeds the safe integer range'
+    );
+  }
+  return value;
+}
+
+function finiteInkCoordinate(value: number): number {
+  if (!Number.isFinite(value)) throw new Error('Ink coordinate is not finite');
+  return value;
+}
+
+function inkBias(
+  value: number | undefined
+): OneNoteInkStroke['bias'] | undefined {
+  if (value === undefined) return 'both';
+  if (value === 0) return 'handwriting';
+  if (value === 1) return 'drawing';
+  if (value === 2) return 'both';
+  return undefined;
+}
+
+function boundingBoxFromStrokes(
+  strokes: readonly OneNoteInkStroke[]
+): OneNoteInkBoundingBox | undefined {
+  let xMin = Infinity;
+  let yMin = Infinity;
+  let xMax = -Infinity;
+  let yMax = -Infinity;
+  for (const stroke of strokes) {
+    for (const point of stroke.points) {
+      xMin = Math.min(xMin, point.x);
+      yMin = Math.min(yMin, point.y);
+      xMax = Math.max(xMax, point.x);
+      yMax = Math.max(yMax, point.y);
+    }
+  }
+  return Number.isFinite(xMin)
+    ? { x: xMin, y: yMin, width: xMax - xMin, height: yMax - yMin }
+    : undefined;
+}
+
+function unionBoundingBoxes(
+  boxes: readonly (OneNoteInkBoundingBox | undefined)[]
+): OneNoteInkBoundingBox | undefined {
+  const present = boxes.filter(
+    (box): box is OneNoteInkBoundingBox => box !== undefined
+  );
+  if (present.length === 0) return undefined;
+  const x = Math.min(...present.map((box) => box.x));
+  const y = Math.min(...present.map((box) => box.y));
+  const x2 = Math.max(...present.map((box) => box.x + box.width));
+  const y2 = Math.max(...present.map((box) => box.y + box.height));
+  return { x, y, width: x2 - x, height: y2 - y };
+}
+
 function parseLayout(object: StoreObject): OneNoteLayout {
   return compactObject({
     maxWidth: optionalF32(object, PROPERTY.LayoutMaxWidth),
@@ -900,6 +1230,17 @@ function incrementTableCellBudget(context: ContentContext): void {
   }
 }
 
+function incrementInkPointBudget(
+  context: ContentContext,
+  amount: number
+): void {
+  const next = context.budget.inkPoints + amount;
+  if (!Number.isSafeInteger(next) || next > context.limits.maxInkPoints) {
+    throw new Error(`Ink point count exceeds ${context.limits.maxInkPoints}`);
+  }
+  context.budget.inkPoints = next;
+}
+
 function addDiagnostic(
   context: ContentContext,
   code: string,

tests/content-model.test.ts

@@ -44,6 +44,24 @@ describe('OneNote rich-content model', () => {
     expect(model.pages[0]!.text).toContain('TEST\tD');
     expect(blocks.some((block) => block.kind === 'table')).toBe(true);
     expect(blocks.some((block) => block.kind === 'image')).toBe(true);
+    const ink = blocks.filter((block) => block.kind === 'ink');
+    expect(ink).toHaveLength(11);
+    expect(
+      ink
+        .flatMap((block) => block.strokes)
+        .reduce((total, stroke) => total + stroke.points.length, 0)
+    ).toBe(1316);
+    expect(ink[0]!.strokes[0]!.points.slice(0, 3)).toEqual([
+      { x: 3360, y: 5848 },
+      { x: 3360, y: 5854 },
+      { x: 3360, y: 5860 },
+    ]);
+    expect(ink[0]!.boundingBox).toEqual({
+      x: 2152,
+      y: 5848,
+      width: 1208,
+      height: 3992,
+    });
 
     const testing = textBlock(blocks, 'Testing!');
     expect(testing.runs.map((run) => run.text)).toEqual(['Testing', '!']);
@@ -86,6 +104,7 @@ describe('OneNote rich-content model', () => {
     const diagnosticCodes = model.diagnostics.map(({ code }) => code);
     expect(diagnosticCodes).not.toContain('CONTENT_PARSE_FAILED');
     expect(diagnosticCodes).not.toContain('INVALID_TABLE_VECTOR');
+    expect(diagnosticCodes).not.toContain('INK_STROKE_PARSE_FAILED');
   });
 
   it('associates only allow-listed hyperlink marker targets', () => {
@@ -101,6 +120,24 @@ describe('OneNote rich-content model', () => {
       unsafe.runs.find((run) => run.text === 'Example')?.href
     ).toBeUndefined();
   });
+
+  it('bounds aggregate ink point expansion', () => {
+    const model = parseOneNoteSectionContent(fixture, {
+      limits: { maxInkPoints: 100 },
+    });
+    expect(model.diagnostics).toContainEqual(
+      expect.objectContaining({
+        code: 'INK_STROKE_PARSE_FAILED',
+        message: expect.stringContaining('Ink point count exceeds 100'),
+      })
+    );
+    const blocks = model.pages.flatMap((page) => collectBlocks(page.blocks));
+    const parsedPoints = blocks
+      .filter((block) => block.kind === 'ink')
+      .flatMap((block) => block.strokes)
+      .reduce((total, stroke) => total + stroke.points.length, 0);
+    expect(parsedPoints).toBeLessThanOrEqual(100);
+  });
 });
 
 function parseSyntheticLink(href: string): OneNoteTextBlock {