Files
onenote-tools/src/onenote/fsshttpb/data-elements.ts

727 lines
21 KiB
TypeScript

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// SPDX-License-Identifier: MPL-2.0
//
// Portions adapted from onenote.rs (MPL-2.0), revision
// 5138a39a3f4e72b840932f9872fecde52fa9da60: `fsshttpb/data_element/*`.
// Deviations: immutable zero-copy payload views, duplicate rejection, exact
// header-length validation, explicit count/byte limits, and offset-aware errors.
import { BinaryReader } from '../binary/BinaryReader.js';
import { readGuid } from '../binary/guid.js';
import type { ExGuid } from '../onestore/types.js';
import { exGuidKey } from '../onestore/types.js';
import { OneNoteParserError } from '../parser/error.js';
import type { OneNoteParserLimits } from '../parser/limits.js';
import {
expectFssHttpEnd,
expectFssHttpStart,
finishFssHttpBody,
hasFssHttpEnd,
readCompactNumber,
readFssHttpCellId,
readFssHttpCellIdArray,
readFssHttpExGuid,
readFssHttpExGuidArray,
readFssHttpSerialNumber,
readFssHttpStreamHeader,
takeFssHttpHeaderBody,
} from './primitives.js';
import {
FSSHTTP_OBJECT,
fssHttpCellKey,
type FssHttpDataElementFragment,
type FssHttpDataElementPackage,
type FssHttpObjectDeclaration,
type FssHttpObjectGroup,
type FssHttpObjectGroupData,
type FssHttpRevisionManifest,
type FssHttpStorageIndex,
type FssHttpStorageManifest,
type FssHttpStreamHeader,
} from './types.js';
interface DataElementContext {
limits: OneNoteParserLimits;
elementCount: number;
objectCount: number;
blobCount: number;
totalBlobBytes: number;
}
export function parseFssHttpDataElementPackage(
reader: BinaryReader,
limits: OneNoteParserLimits
): FssHttpDataElementPackage {
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.dataElementPackage,
true,
2
);
const body = takeFssHttpHeaderBody(
reader,
header,
'Data Element Package start'
);
if (body.u8() !== 0) {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
'Data Element Package reserved byte is not zero',
{ offset: body.offset - 1, structure: 'Data Element Package' }
);
}
finishFssHttpBody(body, 'Data Element Package start');
const result: FssHttpDataElementPackage = {
storageIndexes: new Map(),
storageManifests: new Map(),
cellManifests: new Map(),
revisionManifests: new Map(),
objectGroups: new Map(),
fragments: new Map(),
blobs: new Map(),
};
const context: DataElementContext = {
limits,
elementCount: 0,
objectCount: 0,
blobCount: 0,
totalBlobBytes: 0,
};
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.dataElementPackage)) {
context.elementCount += 1;
if (context.elementCount > limits.maxStreamIds) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`Data element count exceeds ${limits.maxStreamIds}`,
{ offset: reader.offset, structure: 'Data Element Package' }
);
}
parseDataElement(reader, result, context);
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElementPackage);
return result;
}
function parseDataElement(
reader: BinaryReader,
result: FssHttpDataElementPackage,
context: DataElementContext
): void {
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.dataElement,
true,
2
);
const body = takeFssHttpHeaderBody(reader, header, 'Data Element start');
const id = readFssHttpExGuid(body);
readFssHttpSerialNumber(body);
const type = readCompactNumber(body, 0xff, 'Data element type');
finishFssHttpBody(body, 'Data Element start');
switch (type) {
case 0x01:
insertUnique(
result.storageIndexes,
id,
parseStorageIndex(reader, context),
header.offset,
'Storage Index'
);
break;
case 0x02:
insertUnique(
result.storageManifests,
id,
parseStorageManifest(reader, context),
header.offset,
'Storage Manifest'
);
break;
case 0x03:
insertUnique(
result.cellManifests,
id,
parseCellManifest(reader),
header.offset,
'Cell Manifest'
);
break;
case 0x04:
insertUnique(
result.revisionManifests,
id,
parseRevisionManifest(reader, context),
header.offset,
'Revision Manifest'
);
break;
case 0x05:
insertUnique(
result.objectGroups,
id,
parseObjectGroup(reader, context),
header.offset,
'Object Group'
);
break;
case 0x06:
insertUnique(
result.fragments,
id,
parseDataElementFragment(reader, context),
header.offset,
'Data Element Fragment'
);
break;
case 0x0a:
insertUnique(
result.blobs,
id,
parseObjectDataBlob(reader, context),
header.offset,
'Object Data BLOB'
);
break;
default:
throw new OneNoteParserError(
'UNSUPPORTED_FORMAT',
`Unsupported FSSHTTPB data element type 0x${type.toString(16)}`,
{ offset: header.offset, structure: 'Data Element' }
);
}
}
function parseStorageIndex(
reader: BinaryReader,
context: DataElementContext
): FssHttpStorageIndex {
const manifestMappings: FssHttpStorageIndex['manifestMappings'] = [];
const cellMappings: FssHttpStorageIndex['cellMappings'] = new Map();
const revisionMappings: FssHttpStorageIndex['revisionMappings'] = new Map();
let count = 0;
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement)) {
count = boundedIncrement(
count,
context.limits.maxStreamIds,
reader,
'Storage Index mappings'
);
const header = readFssHttpStreamHeader(reader, 2);
if (header.compound) unexpectedCompound(header, 'Storage Index mapping');
const body = takeFssHttpHeaderBody(reader, header, 'Storage Index mapping');
if (header.type === FSSHTTP_OBJECT.storageIndexManifestMapping) {
manifestMappings.push({
id: readFssHttpExGuid(body),
serial: readFssHttpSerialNumber(body),
});
} else if (header.type === FSSHTTP_OBJECT.storageIndexCellMapping) {
const cellId = readFssHttpCellId(body);
const mapping = {
cellId,
id: readFssHttpExGuid(body),
serial: readFssHttpSerialNumber(body),
};
insertUniqueKey(
cellMappings,
fssHttpCellKey(cellId),
mapping,
header.offset,
'Storage Index Cell Mapping'
);
} else if (header.type === FSSHTTP_OBJECT.storageIndexRevisionMapping) {
const id = readFssHttpExGuid(body);
const mapping = {
id,
revision: readFssHttpExGuid(body),
serial: readFssHttpSerialNumber(body),
};
insertUnique(
revisionMappings,
id,
mapping,
header.offset,
'Storage Index Revision Mapping'
);
} else {
unexpectedType(header, 'Storage Index');
}
finishFssHttpBody(body, 'Storage Index mapping');
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return { manifestMappings, cellMappings, revisionMappings };
}
function parseStorageManifest(
reader: BinaryReader,
context: DataElementContext
): FssHttpStorageManifest {
const schemaHeader = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.storageManifest,
false,
2
);
const schemaBody = takeFssHttpHeaderBody(
reader,
schemaHeader,
'Storage Manifest schema'
);
const schema = readGuid(schemaBody);
finishFssHttpBody(schemaBody, 'Storage Manifest schema');
const roots = new Map<string, ReturnType<typeof readFssHttpCellId>>();
let count = 0;
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement)) {
count = boundedIncrement(
count,
context.limits.maxStreamIds,
reader,
'Storage Manifest roots'
);
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.storageManifestRoot,
false,
2
);
const body = takeFssHttpHeaderBody(reader, header, 'Storage Manifest root');
const rootId = readFssHttpExGuid(body);
const cell = readFssHttpCellId(body);
finishFssHttpBody(body, 'Storage Manifest root');
insertUniqueKey(
roots,
exGuidKey(rootId),
cell,
header.offset,
'Storage Manifest root'
);
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return { schema, roots };
}
function parseCellManifest(reader: BinaryReader): ExGuid {
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.cellManifest,
false,
2
);
const body = takeFssHttpHeaderBody(reader, header, 'Cell Manifest');
const revisionId = readFssHttpExGuid(body);
finishFssHttpBody(body, 'Cell Manifest');
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return revisionId;
}
function parseRevisionManifest(
reader: BinaryReader,
context: DataElementContext
): FssHttpRevisionManifest {
const manifestHeader = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.revisionManifest,
false,
2
);
const manifestBody = takeFssHttpHeaderBody(
reader,
manifestHeader,
'Revision Manifest'
);
const revisionId = readFssHttpExGuid(manifestBody);
const baseRevisionId = readFssHttpExGuid(manifestBody);
finishFssHttpBody(manifestBody, 'Revision Manifest');
const roots: FssHttpRevisionManifest['roots'] = [];
const objectGroupIds: ExGuid[] = [];
let count = 0;
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement)) {
count = boundedIncrement(
count,
context.limits.maxStreamIds,
reader,
'Revision Manifest entries'
);
const header = readFssHttpStreamHeader(reader, 2);
if (header.compound) unexpectedCompound(header, 'Revision Manifest entry');
const body = takeFssHttpHeaderBody(
reader,
header,
'Revision Manifest entry'
);
if (header.type === FSSHTTP_OBJECT.revisionManifestRoot) {
roots.push({
role: readFssHttpExGuid(body),
objectId: readFssHttpExGuid(body),
});
} else if (header.type === FSSHTTP_OBJECT.revisionManifestGroupReference) {
objectGroupIds.push(readFssHttpExGuid(body));
} else {
unexpectedType(header, 'Revision Manifest');
}
finishFssHttpBody(body, 'Revision Manifest entry');
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return { revisionId, baseRevisionId, roots, objectGroupIds };
}
function parseObjectGroup(
reader: BinaryReader,
context: DataElementContext
): FssHttpObjectGroup {
const declarationsHeader = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.objectGroupDeclaration,
true
);
const declarationsBody = takeFssHttpHeaderBody(
reader,
declarationsHeader,
'Object Group Declarations start'
);
finishFssHttpBody(declarationsBody, 'Object Group Declarations start');
const declarations: FssHttpObjectDeclaration[] = [];
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.objectGroupDeclaration)) {
context.objectCount += 1;
if (context.objectCount > context.limits.maxObjects) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`FSSHTTPB object declaration count exceeds ${context.limits.maxObjects}`,
{ offset: reader.offset, structure: 'Object Group Declarations' }
);
}
declarations.push(parseObjectDeclaration(reader, context));
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.objectGroupDeclaration);
const changeFrequencies: number[] = [];
let dataHeader = readFssHttpStreamHeader(reader);
if (dataHeader.type === FSSHTTP_OBJECT.objectGroupMetadataBlock) {
if (!dataHeader.compound)
unexpectedCompound(dataHeader, 'Object Group Metadata Block', true);
const metadataBody = takeFssHttpHeaderBody(
reader,
dataHeader,
'Object Group Metadata Block start'
);
finishFssHttpBody(metadataBody, 'Object Group Metadata Block start');
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.objectGroupMetadataBlock)) {
if (changeFrequencies.length >= context.limits.maxObjects) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`Object Group metadata count exceeds ${context.limits.maxObjects}`,
{ offset: reader.offset, structure: 'Object Group Metadata Block' }
);
}
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.objectGroupMetadata,
false,
4
);
const body = takeFssHttpHeaderBody(
reader,
header,
'Object Group Metadata'
);
const frequency = readCompactNumber(body, 4, 'Object change frequency');
changeFrequencies.push(frequency);
finishFssHttpBody(body, 'Object Group Metadata');
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.objectGroupMetadataBlock);
dataHeader = readFssHttpStreamHeader(reader);
}
if (
dataHeader.type !== FSSHTTP_OBJECT.objectGroupData ||
!dataHeader.compound
) {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
`Expected Object Group Data, found type 0x${dataHeader.type.toString(16)}`,
{ offset: dataHeader.offset, structure: 'Object Group' }
);
}
const dataStartBody = takeFssHttpHeaderBody(
reader,
dataHeader,
'Object Group Data start'
);
finishFssHttpBody(dataStartBody, 'Object Group Data start');
const data: FssHttpObjectGroupData[] = [];
while (!hasFssHttpEnd(reader, FSSHTTP_OBJECT.objectGroupData)) {
if (data.length >= context.limits.maxObjects) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`Object Group data count exceeds ${context.limits.maxObjects}`,
{ offset: reader.offset, structure: 'Object Group Data' }
);
}
data.push(parseObjectGroupData(reader, context));
}
expectFssHttpEnd(reader, FSSHTTP_OBJECT.objectGroupData);
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return { declarations, changeFrequencies, data };
}
function parseObjectDeclaration(
reader: BinaryReader,
context: DataElementContext
): FssHttpObjectDeclaration {
const header = readFssHttpStreamHeader(reader);
if (header.compound) unexpectedCompound(header, 'Object declaration');
const body = takeFssHttpHeaderBody(reader, header, 'Object declaration');
const objectId = readFssHttpExGuid(body);
let result: FssHttpObjectDeclaration;
if (header.type === FSSHTTP_OBJECT.objectGroupObject) {
result = {
kind: 'object',
objectId,
partitionId: readCompactNumber(body, 4, 'Object partition ID'),
dataSize: readCompactNumber(
body,
context.limits.maxVectorBytes,
'Object data size'
),
objectReferenceCount: readCompactNumber(
body,
context.limits.maxObjectReferences,
'Object reference count'
),
cellReferenceCount: readCompactNumber(
body,
context.limits.maxObjectReferences,
'Cell reference count'
),
};
} else if (header.type === FSSHTTP_OBJECT.objectGroupDataBlob) {
result = {
kind: 'blob',
objectId,
blobId: readFssHttpExGuid(body),
partitionId: readCompactNumber(body, 4, 'Object partition ID'),
objectReferenceCount: readCompactNumber(
body,
context.limits.maxObjectReferences,
'Object reference count'
),
cellReferenceCount: readCompactNumber(
body,
context.limits.maxObjectReferences,
'Cell reference count'
),
};
} else {
unexpectedType(header, 'Object Group Declarations');
}
finishFssHttpBody(body, 'Object declaration');
return result;
}
function parseObjectGroupData(
reader: BinaryReader,
context: DataElementContext
): FssHttpObjectGroupData {
const header = readFssHttpStreamHeader(reader);
if (header.compound) unexpectedCompound(header, 'Object Group data entry');
const body = takeFssHttpHeaderBody(reader, header, 'Object Group data entry');
const objectReferences = readFssHttpExGuidArray(body, context.limits);
const cellReferences = readFssHttpCellIdArray(body, context.limits);
let result: FssHttpObjectGroupData;
if (header.type === FSSHTTP_OBJECT.objectGroupDataExcluded) {
result = {
kind: 'excluded',
objectReferences,
cellReferences,
dataSize: readCompactNumber(
body,
context.limits.maxVectorBytes,
'Excluded object data size'
),
};
} else if (header.type === FSSHTTP_OBJECT.objectGroupDataObject) {
const size = readCompactNumber(
body,
context.limits.maxVectorBytes,
'Object binary item size'
);
const offset = body.offset;
const data = body.read(size);
result = {
kind: 'object',
objectReferences,
cellReferences,
data,
dataReference: { source: body.bytes, offset, size },
};
} else if (header.type === FSSHTTP_OBJECT.objectGroupBlobReference) {
result = {
kind: 'blob-reference',
objectReferences,
cellReferences,
blobId: readFssHttpExGuid(body),
};
} else {
unexpectedType(header, 'Object Group Data');
}
finishFssHttpBody(body, 'Object Group data entry');
return result;
}
function parseObjectDataBlob(
reader: BinaryReader,
context: DataElementContext
) {
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.objectDataBlob,
false
);
const body = takeFssHttpHeaderBody(reader, header, 'Object Data BLOB');
const size = readCompactNumber(
body,
context.limits.maxFileDataBytes,
'Object Data BLOB size'
);
context.blobCount += 1;
context.totalBlobBytes += size;
if (context.blobCount > context.limits.maxFileDataObjects) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`Object Data BLOB count exceeds ${context.limits.maxFileDataObjects}`,
{ offset: header.offset, structure: 'Object Data BLOB' }
);
}
if (context.totalBlobBytes > context.limits.maxTotalFileDataBytes) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`Object Data BLOB bytes exceed ${context.limits.maxTotalFileDataBytes}`,
{ offset: header.offset, structure: 'Object Data BLOB' }
);
}
const offset = body.offset;
body.skip(size);
finishFssHttpBody(body, 'Object Data BLOB');
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return { source: reader.bytes, offset, size };
}
function parseDataElementFragment(
reader: BinaryReader,
context: DataElementContext
): FssHttpDataElementFragment {
const header = expectFssHttpStart(
reader,
FSSHTTP_OBJECT.dataElementFragment,
false,
4
);
const body = takeFssHttpHeaderBody(reader, header, 'Data Element Fragment');
const dataElementId = readFssHttpExGuid(body);
const fragmentSize = readCompactNumber(
body,
context.limits.maxFileBytes,
'Fragmented data element size'
);
const chunkOffset = readCompactNumber(
body,
fragmentSize,
'Fragment chunk offset'
);
const chunkLength = readCompactNumber(
body,
context.limits.maxVectorBytes,
'Fragment chunk length'
);
if (chunkOffset + chunkLength > fragmentSize) {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
'Fragment chunk range exceeds the fragmented data element size',
{ offset: header.offset, structure: 'Data Element Fragment' }
);
}
if (body.remaining !== chunkLength) {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
`Fragment header declares ${chunkLength} data bytes; ${body.remaining} remain`,
{ offset: header.offset, structure: 'Data Element Fragment' }
);
}
const data = body.read(chunkLength);
finishFssHttpBody(body, 'Data Element Fragment');
expectFssHttpEnd(reader, FSSHTTP_OBJECT.dataElement);
return { dataElementId, fragmentSize, chunkOffset, chunkLength, data };
}
function boundedIncrement(
current: number,
limit: number,
reader: BinaryReader,
structure: string
): number {
const next = current + 1;
if (next > limit) {
throw new OneNoteParserError(
'LIMIT_EXCEEDED',
`${structure} count exceeds ${limit}`,
{ offset: reader.offset, structure }
);
}
return next;
}
function insertUnique<T>(
map: Map<string, T>,
id: ExGuid,
value: T,
offset: number,
structure: string
): void {
insertUniqueKey(map, exGuidKey(id), value, offset, structure);
}
function insertUniqueKey<T>(
map: Map<string, T>,
key: string,
value: T,
offset: number,
structure: string
): void {
if (map.has(key)) {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
`${structure} identifier ${key} is duplicated`,
{ offset, structure }
);
}
map.set(key, value);
}
function unexpectedType(header: FssHttpStreamHeader, structure: string): never {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
`Unexpected stream object type 0x${header.type.toString(16)}`,
{ offset: header.offset, structure }
);
}
function unexpectedCompound(
header: FssHttpStreamHeader,
structure: string,
expected = false
): never {
throw new OneNoteParserError(
'INVALID_STRUCTURE',
`${structure} must be ${expected ? 'compound' : 'single'}`,
{ offset: header.offset, structure }
);
}