onenote-tools/src/onenote/one/section.ts

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
//
// Portions adapted from onenote.rs (MPL-2.0), revision
// 5138a39a3f4e72b840932f9872fecde52fa9da60: `onenote/{section,page_series,
// page,outline,rich_text}.rs` and the corresponding `one/property_set/*.rs`
// modules. Deviations: title/plain-text traversal only, bounded graph walking,
// serializable DTOs, and diagnostics for unsupported content.

import { BinaryReader } from '../binary/BinaryReader.js';
import { filetimeToIso, oneNoteTimeToIso } from '../binary/time.js';
import { appendItems } from '../model/append.js';
import type { ParserDiagnostic } from '../model/diagnostics.js';
import type { OneNotePageSummaryDto, OneNoteSectionDto } from '../model/dto.js';
import type { OneNoteParserLimits } from '../parser/limits.js';
import {
  countObjectReferences,
  countObjectSpaceReferences,
  findProperty,
  referenceOffset,
} from '../onestore/property-set.js';
import type { DesktopOneStore } from '../onestore/desktop-store.js';
import type {
  ExGuid,
  ObjectSpace,
  PropertyEntry,
  StoreObject,
  StoreObjectSpaceReference,
} from '../onestore/types.js';
import { exGuidKey } from '../onestore/types.js';
import { JCID, PROPERTY } from './constants.js';

interface OneContext {
  store: DesktopOneStore;
  limits: OneNoteParserLimits;
  diagnostics: ParserDiagnostic[];
  extractedChars: number;
  currentPage?: string;
}

type PageWithTextDto = OneNotePageSummaryDto & { text: string };

export function buildSectionDto(
  store: DesktopOneStore,
  sourceName: string,
  limits: OneNoteParserLimits,
  diagnostics: ParserDiagnostic[]
): OneNoteSectionDto {
  const context: OneContext = {
    store,
    limits,
    diagnostics,
    extractedChars: 0,
  };
  const root = store.rootObjectSpace;
  const metadata = rootObject(root, 2, 'section metadata root');
  assertJcid(metadata, JCID.SectionMetadata, 'SectionMetadata');
  const displayName = cleanMetadataString(
    optionalString(metadata, PROPERTY.SectionDisplayName)
  );

  const section = rootObject(root, 1, 'section content root');
  assertJcid(section, JCID.SectionNode, 'SectionNode');
  const pageSeriesIds = objectReferences(section, PROPERTY.ElementChildNodes);
  const pages: PageWithTextDto[] = [];

  for (const seriesId of pageSeriesIds) {
    const series = getObject(root, seriesId, 'PageSeriesNode');
    assertJcid(series, JCID.PageSeriesNode, 'PageSeriesNode');
    const pageSpaceIds = objectSpaceReferences(
      series,
      PROPERTY.ChildGraphSpaceElementNodes
    );
    for (const pageSpaceReference of pageSpaceIds) {
      context.currentPage = undefined;
      const pageSpace = store.objectSpaces.get(pageSpaceReference.key);
      if (!pageSpace) {
        addDiagnostic(
          context,
          'MISSING_PAGE_SPACE',
          `Page object space ${pageSpaceReference.key} is missing`,
          'PageSeriesNode'
        );
        continue;
      }

      try {
        if (pages.length >= limits.maxPages) {
          addDiagnostic(
            context,
            'PAGE_LIMIT_REACHED',
            `Page count exceeds ${limits.maxPages}`,
            'PageSeriesNode',
            'error'
          );
          break;
        }
        pages.push(parsePage(context, pageSpace));
      } catch (error) {
        const message = error instanceof Error ? error.message : String(error);
        addDiagnostic(
          context,
          'PAGE_PARSE_FAILED',
          `Could not parse page ${pageSpaceReference.key}: ${message}`,
          'Page',
          'error'
        );
      }
    }
    if (pages.length >= limits.maxPages) break;
  }

  return {
    format: 'one',
    sourceName,
    sectionName: displayName ?? stripOneExtension(sourceName),
    pages,
    diagnostics,
    parserInfo: {
      implementation: 'typescript',
      supportedVariant: store.variant,
      referenceRevision: 'onenote.rs@5138a39a3f4e72b840932f9872fecde52fa9da60',
    },
  };
}

function parsePage(context: OneContext, space: ObjectSpace): PageWithTextDto {
  const metadata = rootObject(space, 2, 'page metadata root');
  assertJcid(metadata, JCID.PageMetadata, 'PageMetadata');
  const pageGuid = requiredGuid(
    metadata,
    PROPERTY.NotebookManagementEntityGuid,
    'PageMetadata entity GUID'
  );
  context.currentPage = pageGuid;
  const created = optionalU64(metadata, PROPERTY.TopologyCreationTimeStamp);
  const level = optionalU32(metadata, PROPERTY.PageLevel);
  const cachedMetadataTitle = optionalString(
    metadata,
    PROPERTY.CachedTitleString
  );

  const manifest = rootObject(space, 1, 'page content root');
  assertJcid(manifest, JCID.PageManifestNode, 'PageManifestNode');
  const pageId = objectReferences(manifest, PROPERTY.ContentChildNodes)[0];
  if (!pageId) throw new Error('PageManifestNode has no page reference');
  const page = getObject(space, pageId, 'PageNode');
  assertJcid(page, JCID.PageNode, 'PageNode');

  const updated = optionalU32(page, PROPERTY.LastModifiedTime);
  const cachedPageTitle = optionalString(
    page,
    PROPERTY.CachedTitleStringFromPage
  );
  const titleId = objectReferences(
    page,
    PROPERTY.StructureElementChildNodes
  )[0];
  const contentIds = objectReferences(page, PROPERTY.ElementChildNodes);

  let titleText: string | undefined;
  if (titleId) {
    try {
      const title = getObject(space, titleId, 'TitleNode');
      assertJcid(title, JCID.TitleNode, 'TitleNode');
      const titleOutlines = objectReferences(title, PROPERTY.ElementChildNodes);
      for (const outlineId of titleOutlines) {
        const paragraphs = extractGraphTextSafely(
          context,
          space,
          outlineId,
          'title'
        );
        titleText = paragraphs.map(cleanTitle).find((item) => item.length > 0);
        if (titleText) break;
      }
    } catch (error) {
      addDiagnostic(
        context,
        'TITLE_PARSE_FAILED',
        error instanceof Error ? error.message : String(error),
        'TitleNode'
      );
    }
  }

  const bodyParagraphs: string[] = [];
  for (const contentId of contentIds) {
    bodyParagraphs.push(
      ...extractGraphTextSafely(context, space, contentId, 'page content')
    );
  }
  let text = bodyParagraphs
    .map((paragraph) => paragraph.split('\u000b').join('\n').trimEnd())
    .filter((paragraph) => paragraph.length > 0)
    .join('\n');
  const remaining =
    context.limits.maxExtractedTextChars - context.extractedChars;
  if (text.length > remaining) {
    text = text.slice(0, Math.max(0, remaining));
    addDiagnostic(
      context,
      'TEXT_LIMIT_REACHED',
      `Extracted text was truncated at ${context.limits.maxExtractedTextChars} characters`,
      `Page ${pageGuid}`
    );
  }
  context.extractedChars += text.length;

  const bodyFallback = bodyParagraphs
    .map(cleanTitle)
    .find((item) => item.length > 0);
  const title =
    titleText ??
    bodyFallback ??
    cachedPageTitle ??
    cachedMetadataTitle ??
    'Untitled page';

  return {
    id: pageGuid,
    title,
    createdAt:
      created === undefined ? undefined : safeFiletime(context, created),
    updatedAt: updated === undefined ? undefined : oneNoteTimeToIso(updated),
    level,
    text,
    textPreview: preview(text),
  };
}

function extractGraphTextSafely(
  context: OneContext,
  space: ObjectSpace,
  id: ExGuid,
  label: string
): string[] {
  try {
    return extractGraphText(context, space, id, new Set(), 0);
  } catch (error) {
    addDiagnostic(
      context,
      'CONTENT_PARSE_FAILED',
      `Could not extract ${label} ${exGuidKey(id)}: ${error instanceof Error ? error.message : String(error)}`,
      'MS-ONE content graph'
    );
    return [];
  }
}

function extractGraphText(
  context: OneContext,
  space: ObjectSpace,
  id: ExGuid,
  active: Set<string>,
  depth: number
): string[] {
  if (depth > context.limits.maxGraphDepth) {
    addDiagnostic(
      context,
      'GRAPH_DEPTH_LIMIT',
      `Content graph depth exceeds ${context.limits.maxGraphDepth}`,
      'MS-ONE content graph'
    );
    return [];
  }
  const key = exGuidKey(id);
  if (active.has(key)) {
    addDiagnostic(
      context,
      'CONTENT_GRAPH_CYCLE',
      `Cycle detected at object ${key}`,
      'MS-ONE content graph'
    );
    return [];
  }

  const object = space.objects.get(key);
  if (!object) {
    addDiagnostic(
      context,
      'MISSING_CONTENT_OBJECT',
      `Content object ${key} is missing`,
      'MS-ONE content graph'
    );
    return [];
  }

  active.add(key);
  try {
    if (object.jcid === JCID.RichTextNode) {
      const text = richText(context, space, object);
      return text === undefined ? [] : [text];
    }
    if (object.jcid === JCID.OutlineNode || object.jcid === JCID.OutlineGroup) {
      return traverseReferences(
        context,
        space,
        objectReferences(object, PROPERTY.ElementChildNodes),
        active,
        depth
      );
    }
    if (object.jcid === JCID.OutlineElementNode) {
      const contents = traverseReferences(
        context,
        space,
        objectReferences(object, PROPERTY.ContentChildNodes),
        active,
        depth
      );
      const children = traverseReferences(
        context,
        space,
        objectReferences(object, PROPERTY.ElementChildNodes),
        active,
        depth
      );
      return [...contents, ...children];
    }

    addDiagnostic(
      context,
      unsupportedCode(object.jcid),
      `Content object JCID 0x${object.jcid.toString(16)} is not yet rendered`,
      `Object ${key}`
    );
    return [];
  } finally {
    active.delete(key);
  }
}

function traverseReferences(
  context: OneContext,
  space: ObjectSpace,
  ids: ExGuid[],
  active: Set<string>,
  depth: number
): string[] {
  const result: string[] = [];
  for (const id of ids) {
    appendItems(
      result,
      extractGraphText(context, space, id, active, depth + 1)
    );
  }
  return result;
}

function richText(
  context: OneContext,
  space: ObjectSpace,
  object: StoreObject
): string | undefined {
  let text = optionalString(object, PROPERTY.RichEditTextUnicode);
  if (text === undefined)
    text = optionalLatin1(object, PROPERTY.TextExtendedAscii);
  if (text === undefined) return undefined;
  const styles = objectReferences(object, PROPERTY.TextRunFormatting);
  if (styles.length === 0) return text.split('\0').join('');
  const indices = optionalU32Vector(object, PROPERTY.TextRunIndex) ?? [];
  const hidden = styles.map((styleId) => {
    const style = space.objects.get(exGuidKey(styleId));
    if (!style) {
      addDiagnostic(
        context,
        'MISSING_TEXT_STYLE',
        `Text-run style ${exGuidKey(styleId)} is missing`,
        'RichTextNode'
      );
      return false;
    }
    return optionalBool(style, PROPERTY.Hidden) ?? false;
  });

  const boundaries = [0, ...indices];
  if (boundaries.length === hidden.length) boundaries.push(text.length);
  let visible = '';
  for (let index = 0; index < hidden.length; index += 1) {
    if (hidden[index]) continue;
    const start = Math.min(boundaries[index] ?? text.length, text.length);
    const end = Math.min(boundaries[index + 1] ?? text.length, text.length);
    if (start < end) visible += text.slice(start, end);
  }
  // OneNote producers sometimes retain UTF-16 terminators inside text vectors.
  return visible.split('\0').join('');
}

function objectReferences(object: StoreObject, propertyId: number): ExGuid[] {
  const property = findProperty(object.props, propertyId);
  if (!property) return [];
  let count: number;
  if (property.value.kind === 'objectId') count = 1;
  else if (property.value.kind === 'objectIds') count = property.value.count;
  else
    throw new Error(
      `Property 0x${propertyId.toString(16)} is not an object reference`
    );

  const offset = referenceOffset(
    object.props,
    propertyId,
    countObjectReferences
  );
  if (object.resolvedObjectIds) {
    return resolvedReferenceSlice(object.resolvedObjectIds, offset, count);
  }
  return resolveReferenceSlice(object, object.props.objectIds, offset, count);
}

function objectSpaceReferences(
  object: StoreObject,
  propertyId: number
): StoreObjectSpaceReference[] {
  const property = findProperty(object.props, propertyId);
  if (!property) return [];
  let count: number;
  if (property.value.kind === 'objectSpaceId') count = 1;
  else if (property.value.kind === 'objectSpaceIds')
    count = property.value.count;
  else {
    throw new Error(
      `Property 0x${propertyId.toString(16)} is not an object-space reference`
    );
  }
  const offset = referenceOffset(
    object.props,
    propertyId,
    countObjectSpaceReferences
  );
  if (object.resolvedObjectSpaceIds) {
    return resolvedReferenceSlice(object.resolvedObjectSpaceIds, offset, count);
  }
  return resolveReferenceSlice(
    object,
    object.props.objectSpaceIds,
    offset,
    count
  ).map((id) => ({ id, key: exGuidKey(id) }));
}

function resolveReferenceSlice(
  object: StoreObject,
  ids: { guidIndex: number; value: number }[],
  offset: number,
  count: number
): ExGuid[] {
  if (offset + count > ids.length) {
    throw new Error(
      `Reference range ${offset}+${count} exceeds stream length ${ids.length}`
    );
  }
  return ids.slice(offset, offset + count).map((id) => {
    const guid = object.mapping.get(id.guidIndex);
    if (!guid) throw new Error(`GUID mapping ${id.guidIndex} is missing`);
    return { guid, value: id.value };
  });
}

function resolvedReferenceSlice<T>(
  ids: readonly T[],
  offset: number,
  count: number
): T[] {
  if (offset + count > ids.length) {
    throw new Error(
      `Reference range ${offset}+${count} exceeds resolved stream length ${ids.length}`
    );
  }
  return ids.slice(offset, offset + count);
}

function rootObject(
  space: ObjectSpace,
  role: number,
  structure: string
): StoreObject {
  const id = space.roots.get(role);
  if (!id) throw new Error(`Object space has no ${structure}`);
  return getObject(space, id, structure);
}

function getObject(
  space: ObjectSpace,
  id: ExGuid,
  structure: string
): StoreObject {
  const object = space.objects.get(exGuidKey(id));
  if (!object)
    throw new Error(`${structure} object ${exGuidKey(id)} is missing`);
  return object;
}

function assertJcid(
  object: StoreObject,
  expected: number,
  structure: string
): void {
  if (object.jcid !== expected) {
    throw new Error(
      `${structure} has JCID 0x${object.jcid.toString(16)}, expected 0x${expected.toString(16)}`
    );
  }
}

function optionalEntry(
  object: StoreObject,
  propertyId: number
): PropertyEntry | undefined {
  return findProperty(object.props, propertyId);
}

function optionalString(
  object: StoreObject,
  propertyId: number
): string | undefined {
  const entry = optionalEntry(object, propertyId);
  if (!entry) return undefined;
  if (entry.value.kind !== 'bytes') {
    throw new Error(
      `Property 0x${propertyId.toString(16)} is not a byte vector`
    );
  }
  if (entry.value.value.byteLength % 2 !== 0) {
    throw new Error(
      `UTF-16 property 0x${propertyId.toString(16)} has odd length`
    );
  }
  return new TextDecoder('utf-16le', { fatal: true }).decode(entry.value.value);
}

function optionalLatin1(
  object: StoreObject,
  propertyId: number
): string | undefined {
  const entry = optionalEntry(object, propertyId);
  if (!entry) return undefined;
  if (entry.value.kind !== 'bytes') {
    throw new Error(
      `Property 0x${propertyId.toString(16)} is not a byte vector`
    );
  }
  let result = '';
  for (const value of entry.value.value) result += String.fromCharCode(value);
  return result;
}

function requiredGuid(
  object: StoreObject,
  propertyId: number,
  structure: string
): string {
  const entry = optionalEntry(object, propertyId);
  if (
    !entry ||
    entry.value.kind !== 'bytes' ||
    entry.value.value.length !== 16
  ) {
    throw new Error(`${structure} is absent or not 16 bytes`);
  }
  const reader = new BinaryReader(entry.value.value, 0, 16, structure);
  const bytes = reader.read(16);
  const order = [3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15];
  const hex = order.map((index) => bytes[index]!.toString(16).padStart(2, '0'));
  return `{${hex.slice(0, 4).join('')}-${hex.slice(4, 6).join('')}-${hex.slice(6, 8).join('')}-${hex.slice(8, 10).join('')}-${hex.slice(10).join('')}}`.toUpperCase();
}

function optionalBool(
  object: StoreObject,
  propertyId: number
): boolean | undefined {
  const value = optionalEntry(object, propertyId)?.value;
  if (!value) return undefined;
  if (value.kind !== 'bool') throw new Error('Expected a Boolean property');
  return value.value;
}

function optionalU32(
  object: StoreObject,
  propertyId: number
): number | undefined {
  const value = optionalEntry(object, propertyId)?.value;
  if (!value) return undefined;
  if (value.kind !== 'u32') throw new Error('Expected a u32 property');
  return value.value;
}

function optionalU64(
  object: StoreObject,
  propertyId: number
): bigint | undefined {
  const value = optionalEntry(object, propertyId)?.value;
  if (!value) return undefined;
  if (value.kind !== 'u64') throw new Error('Expected a u64 property');
  return value.value;
}

function optionalU32Vector(
  object: StoreObject,
  propertyId: number
): number[] | undefined {
  const value = optionalEntry(object, propertyId)?.value;
  if (!value) return undefined;
  if (value.kind !== 'bytes' || value.value.byteLength % 4 !== 0) {
    throw new Error('Expected a u32 vector property');
  }
  const reader = new BinaryReader(
    value.value,
    0,
    value.value.byteLength,
    'u32 vector'
  );
  const result: number[] = [];
  while (reader.remaining > 0) result.push(reader.u32());
  return result;
}

function safeFiletime(context: OneContext, value: bigint): string | undefined {
  try {
    return filetimeToIso(value, 'PageMetadata creation time');
  } catch (error) {
    addDiagnostic(
      context,
      'INVALID_PAGE_TIMESTAMP',
      error instanceof Error ? error.message : String(error),
      'PageMetadata'
    );
    return undefined;
  }
}

function cleanTitle(text: string): string {
  const marker = '\ufddfHYPERLINK "';
  let result = text;
  while (result.startsWith('\u000b')) result = result.slice(1);
  result = result.trim();
  let start = result.indexOf(marker);
  while (start >= 0) {
    const valueStart = start + marker.length;
    const quote = result.indexOf('"', valueStart);
    result =
      quote < 0
        ? result.slice(0, start)
        : result.slice(0, start) + result.slice(quote + 1);
    start = result.indexOf(marker);
  }
  return result.trim();
}

function preview(text: string): string {
  const compact = text.replace(/\s+/g, ' ').trim();
  return compact.length <= 500 ? compact : `${compact.slice(0, 497)}…`;
}

function unsupportedCode(jcid: number): string {
  if (jcid === JCID.ImageNode) return 'UNSUPPORTED_IMAGE';
  if (jcid === JCID.TableNode) return 'UNSUPPORTED_TABLE';
  if (jcid === JCID.EmbeddedFileNode) return 'UNSUPPORTED_EMBEDDED_FILE';
  if (jcid === JCID.InkContainer) return 'UNSUPPORTED_INK';
  return 'UNSUPPORTED_CONTENT_OBJECT';
}

function addDiagnostic(
  context: OneContext,
  code: string,
  message: string,
  structure: string,
  severity: 'warning' | 'error' = 'warning'
): void {
  if (context.diagnostics.length >= context.limits.maxDiagnostics) return;
  context.diagnostics.push({
    severity,
    code,
    message,
    structure: context.currentPage
      ? `${structure} (page ${context.currentPage})`
      : structure,
    recoverable: true,
  });
}

function stripOneExtension(filename: string): string {
  return (filename.split(/[\\/]/).at(-1) ?? filename).replace(/\.one$/i, '');
}

function cleanMetadataString(value: string | undefined): string | undefined {
  const result = value?.split('\0').join('').trim();
  return result || undefined;
}