diff --git a/src/worker/export-adapter.test.ts b/src/worker/export-adapter.test.ts new file mode 100644 index 0000000..12d5e12 --- /dev/null +++ b/src/worker/export-adapter.test.ts @@ -0,0 +1,362 @@ +import { strFromU8, unzipSync } from 'fflate'; +import { describe, expect, it } from 'vitest'; + +import { APP_VERSION } from '../version.js'; +import type { ParserDiagnostic } from '../onenote/model/diagnostics.js'; +import type { + OneNotePackageDto, + OneNotePageDto, + OneNoteSectionDto, +} from '../onenote/model/dto.js'; +import type { OneNoteResource } from '../onenote/one/content-model.js'; +import { + createWorkerExportArtifact, + createWorkerExportSnapshot, + type WorkerExportFormat, +} from './export-adapter.js'; +import { workerPageKey, workerResourceKey } from './parser-adapter.js'; + +const PNG = Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00); + +describe('worker export adapter', () => { + it('builds a partial section snapshot from full session pages and copied resources', () => { + const pageDiagnostic = diagnostic('page-warning', 'page page-1'); + const sectionDiagnostic = diagnostic('section-warning'); + const source = sectionDto('Unsafe/Section.one', 'Section name', [ + pageDiagnostic, + sectionDiagnostic, + ]); + const page = fullPage('page-1', pageDiagnostic); + const resourceSource = Uint8Array.of(7, ...PNG, 9); + const available = resource('image-1', resourceSource, 1, PNG.length); + const attachmentSource = Uint8Array.of(6, 5, 4, 3); + const attachment = resource( + 'attachment-1', + attachmentSource, + 1, + 2, + 'attachment' + ); + const unavailable: OneNoteResource = { + ...resource('missing', resourceSource, 0, 1), + availability: 'missing', + blob: undefined, + }; + + const snapshot = createWorkerExportSnapshot( + source, + new Map([[workerPageKey(page.id), page]]), + new Map([ + [workerResourceKey(available.id), available], + [workerResourceKey(attachment.id), attachment], + [workerResourceKey(unavailable.id), unavailable], + ]) + ); + + expect(snapshot).toMatchObject({ + app: { name: 'OneNote Tools', version: APP_VERSION }, + parser: { + implementation: 'typescript', + supportedVariants: ['desktop-revision-store'], + referenceRevision: 'MS-ONE fixture revision', + }, + supportLevel: 'partial', + source: { name: 'Unsafe/Section.one', format: 'one' }, + notebookName: 'Section name', + }); + expect(snapshot.sections).toHaveLength(1); + expect(snapshot.sections[0]?.pages[0]).toBe(page); + expect(snapshot.sections[0]?.pages[0]?.blocks).toEqual([ + expect.objectContaining({ kind: 'text', text: 'Full rich page body' }), + ]); + expect(snapshot.sections[0]?.diagnostics).toEqual([sectionDiagnostic]); + expect(snapshot.sections[0]?.pages[0]?.diagnostics).toEqual([ + pageDiagnostic, + ]); + expect(snapshot.resources).toHaveLength(2); + expect(snapshot.resources[0]).toMatchObject({ + sectionId: 'section', + id: 'image-1', + mediaType: 'image/png', + size: PNG.length, + }); + expect(new Uint8Array(snapshot.resources[0]!.bytes)).toEqual(PNG); + expect(snapshot.resources[1]).toMatchObject({ + sectionId: 'section', + id: 'attachment-1', + kind: 'attachment', + size: 2, + }); + expect(new Uint8Array(snapshot.resources[1]!.bytes)).toEqual( + Uint8Array.of(5, 4) + ); + resourceSource.fill(0); + attachmentSource.fill(0); + expect(new Uint8Array(snapshot.resources[0]!.bytes)).toEqual(PNG); + expect(new Uint8Array(snapshot.resources[1]!.bytes)).toEqual( + Uint8Array.of(5, 4) + ); + }); + + it('exports only parsed package sections while retaining every diagnostic and the tree', () => { + const packageDiagnostic = diagnostic('cab-warning'); + const parsedWrapperDiagnostic = diagnostic('parsed-wrapper-warning'); + const failedDiagnostic = diagnostic('failed-section-warning'); + const unsupportedDiagnostic = diagnostic('unsupported-section-warning'); + const pageDiagnostic = diagnostic('page-warning', 'page page-1'); + const sectionDiagnostic = diagnostic('section-warning'); + const parsedSection = sectionDto( + 'Folder/Parsed.one', + 'Parsed', + [sectionDiagnostic, pageDiagnostic], + 'fsshttp-packaged-revision-store' + ); + const page = fullPage('page-1', pageDiagnostic); + const source: OneNotePackageDto = { + format: 'onepkg', + sourceName: '../../CON.onepkg', + sourceSize: 123, + entries: [], + sections: [ + { + id: 'Folder/Parsed.one', + path: 'Folder/Parsed.one', + displayName: 'Parsed', + parseStatus: 'parsed', + section: parsedSection, + diagnostics: [parsedWrapperDiagnostic], + }, + { + id: 'Failed.one', + path: 'Failed.one', + displayName: 'Failed', + parseStatus: 'failed', + diagnostics: [failedDiagnostic], + }, + { + id: 'Unsupported.one', + path: 'Unsupported.one', + displayName: 'Unsupported', + parseStatus: 'unsupported', + diagnostics: [unsupportedDiagnostic], + }, + ], + tree: { + interpreted: true, + tocPath: 'Open Notebook.onetoc2', + nodes: [ + { + kind: 'section', + id: 'tree-section', + sectionId: 'Folder/Parsed.one', + displayName: 'Parsed', + }, + ], + }, + diagnostics: [packageDiagnostic], + }; + const bytes = Uint8Array.of(...PNG); + const image = resource('shared-id', bytes, 0, bytes.length); + const wrongSection = resource('other-id', bytes, 0, bytes.length); + + const snapshot = createWorkerExportSnapshot( + source, + new Map([[workerPageKey(page.id, 'Folder/Parsed.one'), page]]), + new Map([ + [workerResourceKey(image.id, 'Folder/Parsed.one'), image], + [workerResourceKey(wrongSection.id, 'Failed.one'), wrongSection], + ]) + ); + + expect(snapshot.source).toEqual({ + name: '../../CON.onepkg', + format: 'onepkg', + }); + expect(snapshot.notebookName).toBe('CON'); + expect(snapshot.tree).toBe(source.tree); + expect(snapshot.sections.map(({ id }) => id)).toEqual([ + 'Folder/Parsed.one', + ]); + expect(snapshot.sections[0]?.pages).toEqual([page]); + expect(snapshot.resources).toEqual([ + expect.objectContaining({ + sectionId: 'Folder/Parsed.one', + id: 'shared-id', + }), + ]); + expect(snapshot.parser.supportedVariants).toEqual([ + 'onepkg-cabinet', + 'fsshttp-packaged-revision-store', + ]); + expect(snapshot.supportNotes.join(' ')).toContain( + '2 package sections were omitted' + ); + const allDiagnostics = [ + ...snapshot.diagnostics, + ...snapshot.sections.flatMap((section) => section.diagnostics), + ...snapshot.sections.flatMap((section) => + section.pages.flatMap((parsedPage) => parsedPage.diagnostics) + ), + ]; + expect(allDiagnostics.map(({ code }) => code)).toEqual([ + 'cab-warning', + 'parsed-wrapper-warning', + 'failed-section-warning', + 'unsupported-section-warning', + 'section-warning', + 'page-warning', + ]); + }); + + it('emits safe, typed text artifacts and a static notebook ZIP', () => { + const source = sectionDto('../../CON.one', undefined, []); + const page = fullPage('page-1'); + page.blocks.push({ + kind: 'image', + id: 'image-block', + resourceId: 'image-1', + filename: 'image-1.png', + isBackground: false, + layout: {}, + }); + const pages = new Map([[workerPageKey(page.id), page]]); + const image = resource('image-1', PNG, 0, PNG.length); + const resources = new Map([[workerResourceKey(image.id), image]]); + const expectations: Array<[WorkerExportFormat, string, string]> = [ + ['json', '_CON.json', 'application/json;charset=utf-8'], + ['text', '_CON.txt', 'text/plain;charset=utf-8'], + ['markdown', '_CON.md', 'text/markdown;charset=utf-8'], + ['html', '_CON.html', 'text/html;charset=utf-8'], + ]; + + for (const [format, filename, mediaType] of expectations) { + const artifact = createWorkerExportArtifact( + source, + pages, + resources, + format + ); + expect(artifact).toMatchObject({ filename, mediaType }); + expect(artifact.bytes).toBeInstanceOf(ArrayBuffer); + expect(artifact.bytes.byteLength).toBeGreaterThan(0); + } + + const json = createWorkerExportArtifact(source, pages, resources, 'json'); + const structured = JSON.parse(new TextDecoder().decode(json.bytes)) as { + supportLevel: string; + sections: Array<{ pages: OneNotePageDto[] }>; + }; + expect(structured.supportLevel).toBe('partial'); + expect(structured.sections[0]?.pages[0]?.blocks[0]).toMatchObject({ + kind: 'text', + text: 'Full rich page body', + }); + + const zip = createWorkerExportArtifact( + source, + pages, + resources, + 'static-zip' + ); + expect(zip).toMatchObject({ + filename: '_CON.zip', + mediaType: 'application/zip', + }); + expect(zip.bytes).toBeInstanceOf(ArrayBuffer); + const files = unzipSync(new Uint8Array(zip.bytes)); + expect(Object.keys(files).sort()).toEqual([ + 'assets/_CON/image-1.png', + 'index.html', + 'manifest.json', + 'notebook.json', + 'notebook.md', + 'notebook.txt', + 'warnings.json', + ]); + expect(strFromU8(files['notebook.txt']!)).toContain('Full rich page body'); + expect(files['assets/_CON/image-1.png']).toEqual(PNG); + }); +}); + +function sectionDto( + sourceName: string, + sectionName: string | undefined, + diagnostics: ParserDiagnostic[], + supportedVariant = 'desktop-revision-store' +): OneNoteSectionDto { + return { + format: 'one', + sourceName, + sectionName, + pages: [ + { + id: 'page-1', + title: 'Summary title', + text: 'Summary-only text', + textPreview: 'Summary-only text', + }, + ], + diagnostics, + parserInfo: { + implementation: 'typescript', + supportedVariant, + referenceRevision: 'MS-ONE fixture revision', + }, + }; +} + +function fullPage( + id: string, + pageDiagnostic?: ParserDiagnostic +): OneNotePageDto { + const text = 'Full rich page body'; + return { + id, + title: 'Full page title', + text, + textPreview: text, + blocks: [ + { + kind: 'text', + id: 'text-1', + sourceText: text, + text, + runs: [{ start: 0, end: text.length, text, style: { bold: true } }], + paragraphStyle: {}, + layout: {}, + }, + ], + diagnostics: pageDiagnostic ? [pageDiagnostic] : [], + }; +} + +function resource( + id: string, + source: Uint8Array, + offset: number, + size: number, + kind: OneNoteResource['kind'] = 'image' +): OneNoteResource { + return { + id, + kind, + sourceJcid: 1, + filename: `${id}.${kind === 'image' ? 'png' : 'bin'}`, + extension: kind === 'image' ? 'png' : 'bin', + mediaType: kind === 'image' ? 'image/png' : 'application/octet-stream', + browserRenderable: kind === 'image', + size, + blob: { source, offset, size }, + availability: 'available', + }; +} + +function diagnostic(code: string, structure?: string): ParserDiagnostic { + return { + severity: 'warning', + code, + message: `${code} message`, + structure, + recoverable: true, + }; +} diff --git a/src/worker/export-adapter.ts b/src/worker/export-adapter.ts new file mode 100644 index 0000000..9b3b672 --- /dev/null +++ b/src/worker/export-adapter.ts @@ -0,0 +1,397 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 + +import { + createOneNoteExportSnapshot, + createOneNoteStaticNotebookZip, + forceFilenameExtension, + sanitizeExportFilename, + serializeOneNoteMarkdown, + serializeOneNotePlainText, + serializeOneNoteSemanticHtml, + serializeOneNoteStructuredJson, + type OneNoteExportOptions, + type OneNoteExportParserMetadata, + type OneNoteExportResource, + type OneNoteExportSection, + type OneNoteExportSnapshot, +} from '../onenote/export/index.js'; +import type { ParserDiagnostic } from '../onenote/model/diagnostics.js'; +import type { + OneNotePackageDto, + OneNotePageDto, + OneNotePageSummaryDto, + OneNoteSectionDto, +} from '../onenote/model/dto.js'; +import type { OneNoteResource } from '../onenote/one/content-model.js'; +import { + resourcePayload, + workerPageKey, + workerResourceKey, +} from './parser-adapter.js'; + +export type WorkerExportFormat = + 'static-zip' | 'json' | 'text' | 'markdown' | 'html'; + +export interface WorkerExportArtifact { + filename: string; + mediaType: string; + bytes: ArrayBuffer; +} + +interface ExportedSectionContext { + exportSection: OneNoteExportSection; + parserSection: OneNoteSectionDto; + sessionSectionId?: string; +} + +interface CollectedPages { + pages: OneNotePageDto[]; + diagnostics: ParserDiagnostic[]; + missingCount: number; +} + +/** Build an inert export snapshot from data retained by one parser session. */ +export function createWorkerExportSnapshot( + source: OneNoteSectionDto | OneNotePackageDto, + pages: ReadonlyMap, + resources: ReadonlyMap +): OneNoteExportSnapshot { + return source.format === 'one' + ? sectionSnapshot(source, pages, resources) + : packageSnapshot(source, pages, resources); +} + +/** Serialize one parser session into a browser-downloadable export artifact. */ +export function createWorkerExportArtifact( + source: OneNoteSectionDto | OneNotePackageDto, + pages: ReadonlyMap, + resources: ReadonlyMap, + format: WorkerExportFormat, + options: OneNoteExportOptions = {} +): WorkerExportArtifact { + const snapshot = createWorkerExportSnapshot(source, pages, resources); + if (format === 'static-zip') { + const artifact = createOneNoteStaticNotebookZip(snapshot, options); + return { + filename: artifact.filename, + mediaType: artifact.mediaType, + bytes: copyToArrayBuffer(artifact.bytes), + }; + } + + const serialization = TEXT_EXPORTS[format]; + const filename = forceFilenameExtension( + sanitizeExportFilename(snapshot.notebookName, 'onenote-export'), + serialization.extension + ); + return { + filename, + mediaType: serialization.mediaType, + bytes: copyToArrayBuffer( + new TextEncoder().encode(serialization.serialize(snapshot, options)) + ), + }; +} + +const TEXT_EXPORTS: Record< + Exclude, + { + extension: string; + mediaType: string; + serialize: ( + snapshot: OneNoteExportSnapshot, + options: OneNoteExportOptions + ) => string; + } +> = { + json: { + extension: 'json', + mediaType: 'application/json;charset=utf-8', + serialize: serializeOneNoteStructuredJson, + }, + text: { + extension: 'txt', + mediaType: 'text/plain;charset=utf-8', + serialize: serializeOneNotePlainText, + }, + markdown: { + extension: 'md', + mediaType: 'text/markdown;charset=utf-8', + serialize: serializeOneNoteMarkdown, + }, + html: { + extension: 'html', + mediaType: 'text/html;charset=utf-8', + serialize: serializeOneNoteSemanticHtml, + }, +}; + +function sectionSnapshot( + source: OneNoteSectionDto, + sessionPages: ReadonlyMap, + sessionResources: ReadonlyMap +): OneNoteExportSnapshot { + const collected = collectPages(source.pages, sessionPages); + const context: ExportedSectionContext = { + parserSection: source, + exportSection: { + id: 'section', + name: source.sectionName ?? filenameStem(source.sourceName, 'Section'), + sourcePath: source.sourceName, + pages: collected.pages, + diagnostics: subtractNestedDiagnostics( + source.diagnostics, + collected.pages + ), + }, + }; + const supportNotes = [PARTIAL_SUPPORT_NOTE]; + if (collected.missingCount > 0) { + supportNotes.push(missingPageSupportNote(collected.missingCount)); + } + + return createOneNoteExportSnapshot({ + sourceName: source.sourceName, + sourceFormat: 'one', + notebookName: + source.sectionName ?? filenameStem(source.sourceName, 'Section'), + parser: parserMetadata([source]), + supportLevel: 'partial', + supportNotes, + sections: [context.exportSection], + resources: collectResources([context], sessionResources), + diagnostics: collected.diagnostics, + }); +} + +function packageSnapshot( + source: OneNotePackageDto, + sessionPages: ReadonlyMap, + sessionResources: ReadonlyMap +): OneNoteExportSnapshot { + const contexts: ExportedSectionContext[] = []; + const globalDiagnostics = [ + ...source.diagnostics, + ...source.sections.flatMap((section) => section.diagnostics), + ]; + let missingPageCount = 0; + + for (const packaged of source.sections) { + if (packaged.parseStatus !== 'parsed' || !packaged.section) continue; + const collected = collectPages( + packaged.section.pages, + sessionPages, + packaged.id, + packaged.path + ); + globalDiagnostics.push(...collected.diagnostics); + missingPageCount += collected.missingCount; + contexts.push({ + sessionSectionId: packaged.id, + parserSection: packaged.section, + exportSection: { + id: packaged.id, + name: packaged.displayName, + sourcePath: packaged.path, + pages: collected.pages, + diagnostics: subtractNestedDiagnostics( + packaged.section.diagnostics, + collected.pages + ), + }, + }); + } + + const skippedCount = source.sections.length - contexts.length; + const supportNotes = [PARTIAL_SUPPORT_NOTE]; + if (skippedCount > 0) { + supportNotes.push( + `${skippedCount} package ${plural(skippedCount, 'section was', 'sections were')} omitted because ${plural(skippedCount, 'it was', 'they were')} not successfully parsed; diagnostics from omitted sections remain included.` + ); + } + if (missingPageCount > 0) { + supportNotes.push(missingPageSupportNote(missingPageCount)); + } + + return createOneNoteExportSnapshot({ + sourceName: source.sourceName, + sourceFormat: 'onepkg', + notebookName: filenameStem(source.sourceName, 'Notebook'), + parser: parserMetadata( + contexts.map(({ parserSection }) => parserSection), + true + ), + supportLevel: 'partial', + supportNotes, + sections: contexts.map(({ exportSection }) => exportSection), + resources: collectResources(contexts, sessionResources), + diagnostics: globalDiagnostics, + tree: source.tree, + }); +} + +const PARTIAL_SUPPORT_NOTE = + 'This export reflects the parser-supported read-only subset; unsupported OneNote structures may remain diagnostic-only.'; + +function parserMetadata( + sections: readonly OneNoteSectionDto[], + packageSource = false +): Partial { + const variants = unique([ + ...(packageSource ? ['onepkg-cabinet'] : []), + ...sections.map(({ parserInfo }) => parserInfo.supportedVariant), + ]); + const revisions = unique( + sections.map(({ parserInfo }) => parserInfo.referenceRevision) + ); + return { + name: 'onenote-tools TypeScript parser', + implementation: 'typescript', + supportedVariants: variants, + ...(revisions.length > 0 + ? { referenceRevision: revisions.join('; ') } + : {}), + }; +} + +function collectPages( + summaries: readonly OneNotePageSummaryDto[], + sessionPages: ReadonlyMap, + sessionSectionId?: string, + sectionPath?: string +): CollectedPages { + const fullPages: OneNotePageDto[] = []; + const diagnostics: ParserDiagnostic[] = []; + for (const summary of summaries) { + const page = sessionPages.get(workerPageKey(summary.id, sessionSectionId)); + if (page) { + fullPages.push(page); + continue; + } + diagnostics.push({ + severity: 'warning', + code: 'export-page-unavailable', + message: `Page ${summary.title || summary.id} was omitted because its full parsed content is no longer available in this session.`, + structure: `${sectionPath ? `${sectionPath}/` : ''}page ${summary.id}`, + recoverable: true, + }); + } + return { + pages: fullPages, + diagnostics, + missingCount: summaries.length - fullPages.length, + }; +} + +function subtractNestedDiagnostics( + sectionDiagnostics: readonly ParserDiagnostic[], + pages: readonly OneNotePageDto[] +): ParserDiagnostic[] { + const nested = new Map(); + for (const diagnostic of pages.flatMap((page) => page.diagnostics)) { + const key = diagnosticKey(diagnostic); + nested.set(key, (nested.get(key) ?? 0) + 1); + } + return sectionDiagnostics.filter((diagnostic) => { + const key = diagnosticKey(diagnostic); + const remaining = nested.get(key) ?? 0; + if (remaining === 0) return true; + nested.set(key, remaining - 1); + return false; + }); +} + +function diagnosticKey(diagnostic: ParserDiagnostic): string { + return JSON.stringify([ + diagnostic.severity, + diagnostic.code, + diagnostic.message, + diagnostic.offset ?? null, + diagnostic.structure ?? null, + diagnostic.propertyId ?? null, + diagnostic.recoverable, + ]); +} + +function collectResources( + sections: readonly ExportedSectionContext[], + sessionResources: ReadonlyMap +): OneNoteExportResource[] { + const result: OneNoteExportResource[] = []; + const sectionBySessionId = new Map( + sections.map((section) => [section.sessionSectionId, section] as const) + ); + for (const [key, resource] of sessionResources) { + const sessionSectionId = resourceSectionId(key, resource.id); + if (sessionSectionId === INVALID_RESOURCE_SECTION) continue; + const context = sectionBySessionId.get(sessionSectionId); + if (!context) continue; + const payload = resourcePayload(resource); + if (!payload) continue; + result.push({ + sectionId: context.exportSection.id, + id: payload.id, + kind: payload.kind, + filename: payload.filename, + extension: payload.extension, + mediaType: payload.mediaType, + browserRenderable: payload.browserRenderable, + size: payload.size, + bytes: payload.bytes, + }); + } + return result; +} + +const INVALID_RESOURCE_SECTION = Symbol('invalid-resource-section'); + +function resourceSectionId( + key: string, + resourceId: string +): string | undefined | typeof INVALID_RESOURCE_SECTION { + let parsed: unknown; + try { + parsed = JSON.parse(key); + } catch { + return INVALID_RESOURCE_SECTION; + } + if ( + !Array.isArray(parsed) || + parsed.length !== 2 || + (parsed[0] !== null && typeof parsed[0] !== 'string') || + parsed[1] !== resourceId + ) { + return INVALID_RESOURCE_SECTION; + } + const sectionId = parsed[0] ?? undefined; + return workerResourceKey(resourceId, sectionId) === key + ? sectionId + : INVALID_RESOURCE_SECTION; +} + +function filenameStem(value: string, fallback: string): string { + const basename = value.split(/[\\/]/u).at(-1)?.split('\0').join('').trim(); + if (!basename) return fallback; + return basename.replace(/\.(?:onepkg|one)$/iu, '') || fallback; +} + +function missingPageSupportNote(count: number): string { + return `${count} ${plural(count, 'page was', 'pages were')} omitted because full parsed content was unavailable in this session.`; +} + +function plural(count: number, singular: string, pluralValue: string): string { + return count === 1 ? singular : pluralValue; +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values.filter((value) => value.trim() !== ''))]; +} + +function copyToArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const result = new ArrayBuffer(bytes.byteLength); + new Uint8Array(result).set(bytes); + return result; +}