Release v0.1.4
This commit is contained in:
@@ -1,4 +1,9 @@
|
||||
type ImportRecord = Record<string, unknown>;
|
||||
type XlsxWorkbookSheet = {
|
||||
sheet: string;
|
||||
data: unknown[][];
|
||||
};
|
||||
type XlsxWorkbookReader = (input: ArrayBuffer) => Promise<XlsxWorkbookSheet[]>;
|
||||
|
||||
export type ImportFieldDefinition = {
|
||||
name: string;
|
||||
@@ -23,13 +28,15 @@ export type CsvParseOptions = {
|
||||
quoted: boolean;
|
||||
};
|
||||
|
||||
export type RecipientImportSourceType = "csv" | "xlsx" | "text";
|
||||
|
||||
export type RecipientColumnKind = "ignore" | "id" | "active" | "name" | "from" | "to" | "cc" | "bcc" | "reply_to" | "field" | "new_field" | "attachment_pattern";
|
||||
|
||||
export type RecipientColumnMapping = {
|
||||
columnIndex: number;
|
||||
kind: RecipientColumnKind;
|
||||
fieldName?: string;
|
||||
newFieldName?: string;
|
||||
fieldName?: string | null;
|
||||
newFieldName?: string | null;
|
||||
};
|
||||
|
||||
export type RecipientImportTable = {
|
||||
@@ -41,6 +48,11 @@ export type RecipientImportTable = {
|
||||
firstDataRowNumber: number;
|
||||
};
|
||||
|
||||
export type RecipientImportSpreadsheetSheet = {
|
||||
name: string;
|
||||
rows: string[][];
|
||||
};
|
||||
|
||||
export type ImportedAddress = {
|
||||
email: string;
|
||||
name?: string;
|
||||
@@ -73,15 +85,84 @@ export type RecipientImportPreview = {
|
||||
patternCount: number;
|
||||
};
|
||||
|
||||
export type RecipientImportColumnProvenance = {
|
||||
column_index: number;
|
||||
header: string;
|
||||
kind: RecipientColumnKind;
|
||||
field_name?: string | null;
|
||||
new_field_name?: string | null;
|
||||
};
|
||||
|
||||
export type RecipientImportProvenance = {
|
||||
id: string;
|
||||
imported_at: string;
|
||||
mode: RecipientImportMode;
|
||||
source_type: RecipientImportSourceType;
|
||||
filename?: string | null;
|
||||
sheet_name?: string | null;
|
||||
encoding?: string | null;
|
||||
delimiter?: string | null;
|
||||
header_rows: number;
|
||||
quoted?: boolean | null;
|
||||
value_separators?: string | null;
|
||||
rows_total: number;
|
||||
valid_rows: number;
|
||||
invalid_rows: number;
|
||||
imported_rows: number;
|
||||
field_names_created: string[];
|
||||
attachment_patterns: number;
|
||||
mapping: RecipientImportColumnProvenance[];
|
||||
};
|
||||
|
||||
export type RecipientImportPreviewOptions = {
|
||||
existingFields: ImportFieldDefinition[];
|
||||
existingEntries?: Record<string, unknown>[];
|
||||
valueSeparators?: string;
|
||||
};
|
||||
|
||||
export type RecipientMappingProfile = {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
columnCount: number;
|
||||
headers: string[];
|
||||
normalizedHeaders: string[];
|
||||
orderedHeaderFingerprint: string;
|
||||
unorderedHeaderFingerprint: string;
|
||||
delimiter: "," | ";" | "\t";
|
||||
headerRows: number;
|
||||
quoted: boolean;
|
||||
valueSeparators: string;
|
||||
mappings: RecipientColumnMapping[];
|
||||
};
|
||||
|
||||
export type RecipientMappingProfileMatchMode = "ordered" | "unordered" | "similar";
|
||||
|
||||
export type RecipientMappingProfileMatch = {
|
||||
profile: RecipientMappingProfile;
|
||||
mode: RecipientMappingProfileMatchMode;
|
||||
score: number;
|
||||
matchedHeaders: number;
|
||||
missingHeaders: string[];
|
||||
extraHeaders: string[];
|
||||
};
|
||||
|
||||
export type RecipientMappingProfileInput = {
|
||||
id?: string;
|
||||
name: string;
|
||||
table: RecipientImportTable;
|
||||
mappings: RecipientColumnMapping[];
|
||||
parseOptions: Pick<CsvParseOptions, "headerRows" | "quoted">;
|
||||
valueSeparators: string;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type MaterializeImportOptions = {
|
||||
mode: RecipientImportMode;
|
||||
attachmentBasePath?: ImportAttachmentBasePath | null;
|
||||
provenance?: RecipientImportProvenance | null;
|
||||
};
|
||||
|
||||
const EMAIL_COLUMNS = new Set(["email", "e_mail", "mail", "to", "to_email", "recipient", "recipient_email"]);
|
||||
@@ -92,13 +173,21 @@ const ACTIVE_COLUMNS = new Set(["active", "enabled", "send", "include"]);
|
||||
export function buildImportTable(text: string, options: CsvParseOptions): RecipientImportTable {
|
||||
const delimiter = options.delimiter === "auto" ? detectDelimiter(text, options.quoted) : options.delimiter;
|
||||
const rows = parseDelimitedText(text, delimiter, options.quoted);
|
||||
return buildImportTableFromRows(rows, { headerRows: options.headerRows, delimiter });
|
||||
}
|
||||
|
||||
export function buildImportTableFromRows(
|
||||
sourceRows: string[][],
|
||||
options: { headerRows: number; delimiter?: "," | ";" | "\t" }
|
||||
): RecipientImportTable {
|
||||
const rows = sourceRows.map((row) => row.map((cell) => String(cell ?? "")));
|
||||
const normalizedHeaderRows = Math.max(0, Math.min(Math.floor(options.headerRows || 0), rows.length));
|
||||
const headerRows = rows.slice(0, normalizedHeaderRows);
|
||||
const dataRows = rows.slice(normalizedHeaderRows);
|
||||
const columnCount = Math.max(0, ...rows.map((row) => row.length));
|
||||
const headers = Array.from({ length: columnCount }, (_value, columnIndex) => headerForColumn(headerRows, columnIndex));
|
||||
return {
|
||||
delimiter,
|
||||
delimiter: options.delimiter ?? ",",
|
||||
headers,
|
||||
headerRows,
|
||||
rows,
|
||||
@@ -107,6 +196,24 @@ export function buildImportTable(text: string, options: CsvParseOptions): Recipi
|
||||
};
|
||||
}
|
||||
|
||||
export async function xlsxSheetsFromArrayBuffer(buffer: ArrayBuffer): Promise<RecipientImportSpreadsheetSheet[]> {
|
||||
const readXlsxFile = await loadXlsxWorkbookReader();
|
||||
const workbook = await readXlsxFile(buffer);
|
||||
return workbook.map((sheet) => ({
|
||||
name: sheet.sheet,
|
||||
rows: sheet.data.map((row) => row.map((cell) => cellToImportText(cell)))
|
||||
}));
|
||||
}
|
||||
|
||||
async function loadXlsxWorkbookReader(): Promise<XlsxWorkbookReader> {
|
||||
if (typeof window !== "undefined" && typeof DOMParser !== "undefined") {
|
||||
const module = await import("read-excel-file/browser");
|
||||
return module.default as XlsxWorkbookReader;
|
||||
}
|
||||
const module = await import("read-excel-file/universal");
|
||||
return module.default as XlsxWorkbookReader;
|
||||
}
|
||||
|
||||
export function defaultColumnMappings(headers: string[], existingFields: ImportFieldDefinition[]): RecipientColumnMapping[] {
|
||||
const existingFieldNames = new Set(existingFields.map((field) => field.name));
|
||||
return headers.map((header, columnIndex) => {
|
||||
@@ -128,6 +235,95 @@ export function defaultColumnMappings(headers: string[], existingFields: ImportF
|
||||
});
|
||||
}
|
||||
|
||||
export function createRecipientMappingProfile(input: RecipientMappingProfileInput): RecipientMappingProfile {
|
||||
const fingerprints = recipientImportHeaderFingerprints(input.table.headers);
|
||||
const updatedAt = input.updatedAt ?? new Date().toISOString();
|
||||
const createdAt = input.createdAt ?? updatedAt;
|
||||
const id = input.id ?? `recipient-import-${fingerprints.orderedHeaderFingerprint}-${Date.now().toString(36)}`;
|
||||
return {
|
||||
id,
|
||||
name: input.name.trim() || "Recipient import mapping",
|
||||
createdAt,
|
||||
updatedAt,
|
||||
columnCount: input.table.headers.length,
|
||||
headers: input.table.headers.slice(),
|
||||
normalizedHeaders: fingerprints.normalizedHeaders,
|
||||
orderedHeaderFingerprint: fingerprints.orderedHeaderFingerprint,
|
||||
unorderedHeaderFingerprint: fingerprints.unorderedHeaderFingerprint,
|
||||
delimiter: input.table.delimiter,
|
||||
headerRows: Math.max(0, Math.floor(input.parseOptions.headerRows || 0)),
|
||||
quoted: input.parseOptions.quoted,
|
||||
valueSeparators: input.valueSeparators,
|
||||
mappings: normalizeMappingList(input.mappings, input.table.headers.length)
|
||||
};
|
||||
}
|
||||
|
||||
export function recipientImportHeaderFingerprints(headers: string[]): Pick<RecipientMappingProfile, "normalizedHeaders" | "orderedHeaderFingerprint" | "unorderedHeaderFingerprint"> {
|
||||
const normalizedHeaders = headers.map(normalizeImportHeader);
|
||||
return {
|
||||
normalizedHeaders,
|
||||
orderedHeaderFingerprint: fingerprintHeaderList(normalizedHeaders),
|
||||
unorderedHeaderFingerprint: fingerprintHeaderList(normalizedHeaders.slice().sort())
|
||||
};
|
||||
}
|
||||
|
||||
export function matchRecipientMappingProfiles(table: RecipientImportTable, profiles: RecipientMappingProfile[]): RecipientMappingProfileMatch[] {
|
||||
const fingerprints = recipientImportHeaderFingerprints(table.headers);
|
||||
return profiles
|
||||
.map((profile): RecipientMappingProfileMatch | null => {
|
||||
const diff = headerSetDiff(fingerprints.normalizedHeaders, profile.normalizedHeaders);
|
||||
const exactOrdered = fingerprints.orderedHeaderFingerprint === profile.orderedHeaderFingerprint;
|
||||
const exactUnordered = fingerprints.unorderedHeaderFingerprint === profile.unorderedHeaderFingerprint;
|
||||
const mode: RecipientMappingProfileMatchMode = exactOrdered ? "ordered" : exactUnordered ? "unordered" : "similar";
|
||||
const baseScore = exactOrdered ? 1 : exactUnordered ? 0.96 : diff.matchedHeaders / Math.max(fingerprints.normalizedHeaders.length, profile.normalizedHeaders.length, 1);
|
||||
const mappedMissingCount = missingMappedHeaders(profile, fingerprints.normalizedHeaders).length;
|
||||
const score = mode === "similar" ? Math.max(0, baseScore - Math.min(0.25, mappedMissingCount * 0.05)) : baseScore;
|
||||
if (mode === "similar" && score < 0.45) return null;
|
||||
return {
|
||||
profile,
|
||||
mode,
|
||||
score,
|
||||
matchedHeaders: diff.matchedHeaders,
|
||||
missingHeaders: diff.missingHeaders,
|
||||
extraHeaders: diff.extraHeaders
|
||||
};
|
||||
})
|
||||
.filter((match): match is RecipientMappingProfileMatch => match !== null)
|
||||
.sort((left, right) => {
|
||||
if (right.score !== left.score) return right.score - left.score;
|
||||
return right.profile.updatedAt.localeCompare(left.profile.updatedAt);
|
||||
});
|
||||
}
|
||||
|
||||
export function applyRecipientMappingProfile(
|
||||
table: RecipientImportTable,
|
||||
profile: RecipientMappingProfile,
|
||||
existingFields: ImportFieldDefinition[]
|
||||
): RecipientColumnMapping[] {
|
||||
const defaults = defaultColumnMappings(table.headers, existingFields);
|
||||
const fingerprints = recipientImportHeaderFingerprints(table.headers);
|
||||
const profileMappings = normalizeMappingList(profile.mappings, profile.columnCount || profile.headers.length);
|
||||
const exactOrdered = fingerprints.orderedHeaderFingerprint === profile.orderedHeaderFingerprint && table.headers.length === profileMappings.length;
|
||||
|
||||
if (exactOrdered) {
|
||||
return table.headers.map((header, columnIndex) => normalizeProfileMapping(profileMappings[columnIndex], columnIndex, header, existingFields, defaults[columnIndex]));
|
||||
}
|
||||
|
||||
const profileIndexesByHeader = new Map<string, number[]>();
|
||||
profile.normalizedHeaders.forEach((header, index) => {
|
||||
const indexes = profileIndexesByHeader.get(header) ?? [];
|
||||
indexes.push(index);
|
||||
profileIndexesByHeader.set(header, indexes);
|
||||
});
|
||||
|
||||
return table.headers.map((header, columnIndex) => {
|
||||
const normalizedHeader = normalizeImportHeader(header);
|
||||
const profileIndex = profileIndexesByHeader.get(normalizedHeader)?.shift();
|
||||
if (profileIndex === undefined) return defaults[columnIndex] ?? { columnIndex, kind: "ignore" };
|
||||
return normalizeProfileMapping(profileMappings[profileIndex], columnIndex, header, existingFields, defaults[columnIndex]);
|
||||
});
|
||||
}
|
||||
|
||||
export function buildRecipientImportPreview(
|
||||
table: RecipientImportTable,
|
||||
mappings: RecipientColumnMapping[],
|
||||
@@ -240,11 +436,13 @@ export function materializeRecipientImport(
|
||||
): Record<string, unknown> {
|
||||
const currentEntries = asRecord(draft.entries);
|
||||
const currentInlineEntries = asArray(currentEntries.inline).map(asRecord);
|
||||
const previousImports = asArray(currentEntries.imports).map(asRecord);
|
||||
const importedEntries = preview.rows
|
||||
.filter((row) => row.issues.length === 0)
|
||||
.map((row) => importedRowToEntry(row, options.attachmentBasePath));
|
||||
const nextInlineEntries = options.mode === "append" ? [...currentInlineEntries, ...importedEntries] : importedEntries;
|
||||
const nextEntries: ImportRecord = { ...currentEntries, inline: nextInlineEntries };
|
||||
if (options.provenance) nextEntries.imports = [...previousImports, options.provenance];
|
||||
delete nextEntries.source;
|
||||
delete nextEntries.mapping;
|
||||
|
||||
@@ -255,6 +453,52 @@ export function materializeRecipientImport(
|
||||
};
|
||||
}
|
||||
|
||||
export function createRecipientImportProvenance(input: {
|
||||
preview: RecipientImportPreview;
|
||||
mappings: RecipientColumnMapping[];
|
||||
mode: RecipientImportMode;
|
||||
sourceType: RecipientImportSourceType;
|
||||
filename?: string | null;
|
||||
sheetName?: string | null;
|
||||
encoding?: string | null;
|
||||
delimiter?: string | null;
|
||||
headerRows: number;
|
||||
quoted?: boolean | null;
|
||||
valueSeparators?: string | null;
|
||||
}): RecipientImportProvenance {
|
||||
const now = new Date().toISOString();
|
||||
const mappingsByColumn = new Map(input.mappings.map((mapping) => [mapping.columnIndex, mapping]));
|
||||
return {
|
||||
id: `recipient-import-${stableHash(`${now}\u001f${input.filename ?? ""}\u001f${input.sheetName ?? ""}\u001f${input.preview.rows.length}`)}`,
|
||||
imported_at: now,
|
||||
mode: input.mode,
|
||||
source_type: input.sourceType,
|
||||
filename: input.filename?.trim() || null,
|
||||
sheet_name: input.sheetName?.trim() || null,
|
||||
encoding: input.encoding?.trim() || null,
|
||||
delimiter: input.delimiter?.trim() || null,
|
||||
header_rows: Math.max(0, Math.floor(input.headerRows || 0)),
|
||||
quoted: input.quoted ?? null,
|
||||
value_separators: input.valueSeparators ?? null,
|
||||
rows_total: input.preview.rows.length,
|
||||
valid_rows: input.preview.validCount,
|
||||
invalid_rows: input.preview.invalidCount,
|
||||
imported_rows: input.preview.validCount,
|
||||
field_names_created: input.preview.fieldNamesToCreate.slice(),
|
||||
attachment_patterns: input.preview.patternCount,
|
||||
mapping: input.preview.table.headers.map((header, columnIndex) => {
|
||||
const mapping = mappingsByColumn.get(columnIndex) ?? { columnIndex, kind: "ignore" as const };
|
||||
return {
|
||||
column_index: columnIndex,
|
||||
header,
|
||||
kind: mapping.kind,
|
||||
...(mapping.fieldName ? { field_name: mapping.fieldName } : {}),
|
||||
...(mapping.newFieldName ? { new_field_name: mapping.newFieldName } : {})
|
||||
};
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
export function importedRowsNeedAttachmentSource(preview: RecipientImportPreview): boolean {
|
||||
return preview.rows.some((row) => row.issues.length === 0 && row.patterns.length > 0);
|
||||
}
|
||||
@@ -360,6 +604,12 @@ function headerForColumn(headerRows: string[][], columnIndex: number): string {
|
||||
return parts.length ? parts.join(" / ") : `Column ${columnIndex + 1}`;
|
||||
}
|
||||
|
||||
function cellToImportText(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function explicitFieldNameFromHeader(header: string): string {
|
||||
const explicit = /^(?:field|fields)[.:](.+)$/i.exec(header.trim());
|
||||
return explicit ? sanitizeFieldName(explicit[1]) : "";
|
||||
@@ -369,6 +619,93 @@ function normalizeColumnKey(value: string): string {
|
||||
return value.trim().toLowerCase().replace(/[\s.-]+/g, "_").replace(/^_+|_+$/g, "");
|
||||
}
|
||||
|
||||
function normalizeImportHeader(value: string): string {
|
||||
return normalizeColumnKey(value).replace(/[^a-z0-9_]+/g, "_").replace(/_+/g, "_").replace(/^_+|_+$/g, "");
|
||||
}
|
||||
|
||||
function fingerprintHeaderList(headers: string[]): string {
|
||||
return stableHash(headers.join("\u001f"));
|
||||
}
|
||||
|
||||
function stableHash(value: string): string {
|
||||
let hash = 2166136261;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
hash ^= value.charCodeAt(index);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return (hash >>> 0).toString(36);
|
||||
}
|
||||
|
||||
function headerSetDiff(currentHeaders: string[], profileHeaders: string[]): { matchedHeaders: number; missingHeaders: string[]; extraHeaders: string[] } {
|
||||
const currentCounts = countHeaders(currentHeaders);
|
||||
const profileCounts = countHeaders(profileHeaders);
|
||||
let matchedHeaders = 0;
|
||||
const missingHeaders: string[] = [];
|
||||
const extraHeaders: string[] = [];
|
||||
|
||||
for (const [header, profileCount] of profileCounts) {
|
||||
const currentCount = currentCounts.get(header) ?? 0;
|
||||
matchedHeaders += Math.min(currentCount, profileCount);
|
||||
for (let index = currentCount; index < profileCount; index += 1) missingHeaders.push(header);
|
||||
}
|
||||
for (const [header, currentCount] of currentCounts) {
|
||||
const profileCount = profileCounts.get(header) ?? 0;
|
||||
for (let index = profileCount; index < currentCount; index += 1) extraHeaders.push(header);
|
||||
}
|
||||
|
||||
return { matchedHeaders, missingHeaders, extraHeaders };
|
||||
}
|
||||
|
||||
function countHeaders(headers: string[]): Map<string, number> {
|
||||
const counts = new Map<string, number>();
|
||||
headers.forEach((header) => counts.set(header, (counts.get(header) ?? 0) + 1));
|
||||
return counts;
|
||||
}
|
||||
|
||||
function missingMappedHeaders(profile: RecipientMappingProfile, currentHeaders: string[]): string[] {
|
||||
const currentCounts = countHeaders(currentHeaders);
|
||||
return normalizeMappingList(profile.mappings, profile.columnCount || profile.headers.length)
|
||||
.filter((mapping) => mapping.kind !== "ignore")
|
||||
.map((mapping) => profile.normalizedHeaders[mapping.columnIndex] ?? "")
|
||||
.filter((header) => header && !currentCounts.has(header));
|
||||
}
|
||||
|
||||
function normalizeMappingList(mappings: RecipientColumnMapping[], columnCount: number): RecipientColumnMapping[] {
|
||||
const byColumn = new Map(mappings.map((mapping) => [mapping.columnIndex, mapping]));
|
||||
return Array.from({ length: Math.max(0, columnCount) }, (_value, columnIndex) => normalizeMappingShape(byColumn.get(columnIndex), columnIndex));
|
||||
}
|
||||
|
||||
function normalizeMappingShape(mapping: RecipientColumnMapping | undefined, columnIndex: number): RecipientColumnMapping {
|
||||
if (!mapping) return { columnIndex, kind: "ignore" };
|
||||
const kind = mapping.kind;
|
||||
if (kind === "field") return { columnIndex, kind, fieldName: mapping.fieldName ?? "" };
|
||||
if (kind === "new_field") return { columnIndex, kind, newFieldName: mapping.newFieldName ?? "" };
|
||||
return { columnIndex, kind };
|
||||
}
|
||||
|
||||
function normalizeProfileMapping(
|
||||
mapping: RecipientColumnMapping | undefined,
|
||||
columnIndex: number,
|
||||
header: string,
|
||||
existingFields: ImportFieldDefinition[],
|
||||
fallback: RecipientColumnMapping | undefined
|
||||
): RecipientColumnMapping {
|
||||
const fallbackMapping = fallback ? { ...fallback, columnIndex } : { columnIndex, kind: "ignore" as const };
|
||||
if (!mapping) return fallbackMapping;
|
||||
const existingFieldNames = new Set(existingFields.map((field) => field.name));
|
||||
if (mapping.kind === "field") {
|
||||
const fieldName = sanitizeFieldName(mapping.fieldName ?? "");
|
||||
if (!fieldName) return fallbackMapping;
|
||||
return existingFieldNames.has(fieldName) ? { columnIndex, kind: "field", fieldName } : { columnIndex, kind: "new_field", newFieldName: fieldName };
|
||||
}
|
||||
if (mapping.kind === "new_field") {
|
||||
const fieldName = sanitizeFieldName(mapping.newFieldName || header);
|
||||
if (!fieldName) return fallbackMapping;
|
||||
return existingFieldNames.has(fieldName) ? { columnIndex, kind: "field", fieldName } : { columnIndex, kind: "new_field", newFieldName: fieldName };
|
||||
}
|
||||
return { columnIndex, kind: mapping.kind };
|
||||
}
|
||||
|
||||
function isPatternColumn(key: string): boolean {
|
||||
return key === "pattern"
|
||||
|| key === "patterns"
|
||||
|
||||
Reference in New Issue
Block a user