diff --git a/docs/ADDRESS_MODULE_ARCHITECTURE.md b/docs/ADDRESS_MODULE_ARCHITECTURE.md index 05d2cb8..2c0b169 100644 --- a/docs/ADDRESS_MODULE_ARCHITECTURE.md +++ b/docs/ADDRESS_MODULE_ARCHITECTURE.md @@ -187,9 +187,9 @@ successful search can infer deletion. A timeout, bind failure, malformed entry, duplicate key, or configured entry limit retains existing contacts and reports the source as failed/stale instead of creating tombstones. -## Static Tabular Imports +## Static Mapped Imports -CSV and XLSX use versioned, scoped mapping profiles rather than live sync +CSV, XLSX, and LDIF use versioned, scoped mapping profiles rather than live sync sources. Profiles retain delimiter, encoding, header or worksheet selection, stable source-key mapping, field mappings, locale and tags, row limits, and explicit duplicate, blank-value, and existing-contact policies. Updating a @@ -207,6 +207,15 @@ matches its recorded post-apply hash. Arbitrary transforms remain Dataflow's responsibility; Files and Datasources are optional origins, not prerequisites for direct upload. +LDIF is unfolded and parsed as a bounded stream of entries. Attribute names are +case-insensitive; UTF-8 and base64-encoded text and repeated values are retained +for mapping, while binary and URL values produce diagnostics and are never +projected or fetched. The default change-record policy rejects change records. +A profile can instead ignore them, or treat `changetype: add` as a static entry; +modify and delete records are never translated into contact mutations. Entry +hashes, input/plan hashes, mapping-version provenance, and the same correction +and guarded rollback lifecycle apply as for CSV and XLSX. + Persisted import runs can be resumed through `/address-book?import_run=`. The WebUI reloads the bounded run projection, selects its address book and mapping version, and restores statistics, diagnostics, effects, and lifecycle diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index a340d2f..d7d82d3 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -224,7 +224,7 @@ Tasks: - [ ] [On-premises Exchange connector profile](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/17) - [ ] [Google People contacts connector](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/18) - [ ] [Reusable CSV/XLSX import mapping profiles](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/19) -- [ ] [Bounded LDIF import profile](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/20) +- [x] [Bounded LDIF import profile](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/20) - [ ] [Selective and large-batch vCard workflows](https://git.add-ideas.de/GovOPlaN/govoplan-addresses/issues/21) - [x] classical address-list UI; reusable static/dynamic operational segments move to `govoplan-dist-lists` diff --git a/src/govoplan_addresses/backend/import_schemas.py b/src/govoplan_addresses/backend/import_schemas.py index d0ab93e..9c77c45 100644 --- a/src/govoplan_addresses/backend/import_schemas.py +++ b/src/govoplan_addresses/backend/import_schemas.py @@ -6,7 +6,7 @@ from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator -AddressImportFormat = Literal["csv", "xlsx"] +AddressImportFormat = Literal["csv", "xlsx", "ldif"] AddressImportScope = Literal["user", "group", "tenant", "system"] IMPORT_TARGET_FIELDS = frozenset( @@ -41,6 +41,7 @@ class AddressImportConfiguration(BaseModel): duplicate_source_key_policy: Literal["reject", "first", "last"] = "reject" existing_contact_policy: Literal["update", "ignore", "reject"] = "update" blank_value_policy: Literal["ignore", "clear", "reject"] = "ignore" + ldif_change_record_policy: Literal["reject", "ignore", "treat_add_as_entry"] = "reject" locale: str | None = Field(default=None, max_length=35) default_tags: list[str] = Field(default_factory=list, max_length=100) max_rows: int = Field(default=10_000, ge=1, le=10_000) diff --git a/src/govoplan_addresses/backend/imports.py b/src/govoplan_addresses/backend/imports.py index 17f4fee..67509ed 100644 --- a/src/govoplan_addresses/backend/imports.py +++ b/src/govoplan_addresses/backend/imports.py @@ -24,6 +24,7 @@ from govoplan_addresses.backend.import_schemas import ( AddressImportProfileUpdateRequest, AddressImportRollbackRequest, ) +from govoplan_addresses.backend.ldif import parse_ldif_rows from govoplan_addresses.backend.schemas import ( ContactCreateRequest, ContactEmailPayload, @@ -199,6 +200,7 @@ def preview_address_import( input_hash=input_hash, rows=rows, config=config, + source_format=profile.source_format, ) diagnostics = [*parse_diagnostics, *map_diagnostics] statistics = dict(Counter(item["action"] for item in plan_data)) @@ -501,13 +503,24 @@ def _parse_rows( filename: str, source_format: str, config: AddressImportConfiguration, -) -> tuple[list[tuple[int, dict[str, str]]], list[dict[str, Any]]]: +) -> tuple[list[tuple[int, dict[str, Any]]], list[dict[str, Any]]]: if source_format == "csv": return _parse_csv(raw, config=config) if source_format == "xlsx": if not filename.casefold().endswith(".xlsx"): raise AddressBookError("XLSX imports require an .xlsx file; macros and legacy workbooks are not accepted.") return _parse_xlsx(raw, config=config) + if source_format == "ldif": + if not filename.casefold().endswith((".ldif", ".ldi")): + raise AddressBookError("LDIF imports require an .ldif or .ldi file.") + try: + return parse_ldif_rows( + raw, + max_entries=config.max_rows, + change_record_policy=config.ldif_change_record_policy, + ) + except ValueError as exc: + raise AddressBookError(str(exc)) from exc raise AddressBookError(f"Unsupported address import format: {source_format!r}.") @@ -600,32 +613,81 @@ def _row_dict(headers: list[str], values: list[Any]) -> dict[str, str]: } +def _column_key(value: object, *, casefold: bool) -> str: + column = str(value).strip() + return column.casefold() if casefold else column + + +def _row_value(row: dict[str, Any], column: str, *, casefold: bool) -> object: + if column in row: + return row[column] + if not casefold: + return "" + expected = column.casefold() + for key, value in row.items(): + if key.casefold() == expected: + return value + return "" + + +def _import_values(value: object) -> list[str]: + raw_values = value if isinstance(value, list) else [value] + return [str(item).strip() for item in raw_values if item is not None and str(item).strip()] + + +def _first_import_value(value: object) -> str: + values = _import_values(value) + return values[0] if values else "" + + def _plan_rows( session: Session, *, book_id: str, profile: AddressImportProfile, input_hash: str, - rows: list[tuple[int, dict[str, str]]], + rows: list[tuple[int, dict[str, Any]]], config: AddressImportConfiguration, + source_format: str, ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: diagnostics: list[dict[str, Any]] = [] plan: list[dict[str, Any]] = [] - headers = set(rows[0][1]) if rows else set() - referenced_columns = set(config.field_mappings.values()) + casefold_columns = source_format == "ldif" + headers = { + _column_key(column, casefold=casefold_columns) + for column in (rows[0][1] if rows else set()) + if not str(column).startswith("__") + } + referenced_columns = { + _column_key(column, casefold=casefold_columns) + for column in config.field_mappings.values() + } + key_column = config.source_key_column or config.field_mappings["source_key"] + normalized_key_column = _column_key(key_column, casefold=casefold_columns) if config.source_key_column: - referenced_columns.add(config.source_key_column) + referenced_columns.add(_column_key(config.source_key_column, casefold=casefold_columns)) missing_columns = sorted(referenced_columns.difference(headers)) for column in missing_columns: - diagnostics.append(_diagnostic("error", "missing_column", f'Configured column "{column}" is missing.', field=column)) - if missing_columns: + optional_ldif_attribute = casefold_columns and column != normalized_key_column + diagnostics.append( + _diagnostic( + "warning" if optional_ldif_attribute else "error", + "missing_attribute" if optional_ldif_attribute else "missing_column", + ( + f'Configured LDIF attribute "{column}" is absent from this file.' + if optional_ldif_attribute + else f'Configured column "{column}" is missing.' + ), + field=column, + ) + ) + if normalized_key_column in missing_columns or (missing_columns and not casefold_columns): return [], diagnostics - key_column = config.source_key_column or config.field_mappings["source_key"] - keyed_rows: list[tuple[int, dict[str, str], str]] = [] + keyed_rows: list[tuple[int, dict[str, Any], str]] = [] key_counts: Counter[str] = Counter() for row_number, row in rows: - key = row.get(key_column, "").strip() + key = _first_import_value(_row_value(row, key_column, casefold=casefold_columns)).strip() if not key: diagnostics.append(_diagnostic("error", "missing_source_key", "Stable source key is blank.", row_number=row_number, field=key_column)) plan.append(_plan_effect(row_number, "conflict", source_key=None, message="Stable source key is blank.")) @@ -651,11 +713,23 @@ def _plan_rows( plan.append(_plan_effect(row_number, "ignored", source_key=key, message="Duplicate row ignored by profile policy.")) continue - mapped, row_diagnostics = _mapped_fields(row_number, row, config=config) + mapped, row_diagnostics = _mapped_fields( + row_number, + row, + config=config, + casefold_columns=casefold_columns, + ) diagnostics.extend(row_diagnostics) source_ref = f"import:{profile.profile_key}:{key}" existing = _contact_by_source_ref(session, book_id, source_ref) - payload = _payload_from_mapped(mapped, profile=profile, input_hash=input_hash, row_number=row_number, source_key=key) + payload = _payload_from_mapped( + mapped, + profile=profile, + input_hash=input_hash, + row_number=row_number, + source_key=key, + source_row=row, + ) display_name = payload.get("display_name") or payload.get("email") or key if any(item["severity"] == "error" for item in row_diagnostics): plan.append(_plan_effect(row_number, "conflict", source_key=key, display_name=display_name, source_ref=source_ref, payload=payload, message="Row validation failed.")) @@ -689,23 +763,46 @@ def _plan_rows( def _mapped_fields( row_number: int, - row: dict[str, str], + row: dict[str, Any], *, config: AddressImportConfiguration, + casefold_columns: bool = False, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: mapped: dict[str, Any] = {} diagnostics: list[dict[str, Any]] = [] for target, column in config.field_mappings.items(): if target == "source_key": continue - value = row.get(column, "").strip() - if not value: + values = _import_values(_row_value(row, column, casefold=casefold_columns)) + if not values: if config.blank_value_policy == "reject": diagnostics.append(_diagnostic("error", "blank_value", f'Column "{column}" is blank.', row_number=row_number, field=target)) elif config.blank_value_policy == "clear": mapped[target] = [] if target == "tags" else None continue - mapped[target] = [item.strip() for item in value.split(",") if item.strip()] if target == "tags" else value + if target == "tags": + mapped[target] = list( + dict.fromkeys( + item.strip() + for value in values + for item in value.split(",") + if item.strip() + ) + ) + elif target in {"email", "phone"}: + mapped[target] = values if len(values) > 1 else values[0] + else: + mapped[target] = values[0] + if len(values) > 1: + diagnostics.append( + _diagnostic( + "warning", + "multiple_values_truncated", + f'Attribute "{column}" has multiple values; only the first maps to {target!r}.', + row_number=row_number, + field=target, + ) + ) if config.default_tags: mapped["tags"] = list(dict.fromkeys([*(mapped.get("tags") or []), *config.default_tags])) if not any(mapped.get(name) for name in ("display_name", "given_name", "family_name", "email", "organization")): @@ -720,8 +817,22 @@ def _payload_from_mapped( input_hash: str, row_number: int, source_key: str, + source_row: dict[str, Any] | None = None, ) -> dict[str, Any]: - display_name = mapped.get("display_name") or " ".join(filter(None, [mapped.get("given_name"), mapped.get("family_name")])) or mapped.get("email") or mapped.get("organization") + display_name = ( + _first_import_value(mapped.get("display_name")) + or " ".join( + filter( + None, + [ + _first_import_value(mapped.get("given_name")), + _first_import_value(mapped.get("family_name")), + ], + ) + ) + or _first_import_value(mapped.get("email")) + or _first_import_value(mapped.get("organization")) + ) payload: dict[str, Any] = { key: mapped.get(key) for key in ("given_name", "family_name", "organization", "role_title", "note", "tags") @@ -729,9 +840,17 @@ def _payload_from_mapped( } payload["display_name"] = display_name if "email" in mapped: - payload["emails"] = [] if mapped["email"] is None else [ContactEmailPayload(email=mapped["email"], is_primary=True).model_dump(mode="json")] + email_values = _import_values(mapped["email"]) + payload["emails"] = [ + ContactEmailPayload(email=value, is_primary=index == 0).model_dump(mode="json") + for index, value in enumerate(email_values) + ] if "phone" in mapped: - payload["phones"] = [] if mapped["phone"] is None else [ContactPhonePayload(phone=mapped["phone"], is_primary=True).model_dump(mode="json")] + phone_values = _import_values(mapped["phone"]) + payload["phones"] = [ + ContactPhonePayload(phone=value, is_primary=index == 0).model_dump(mode="json") + for index, value in enumerate(phone_values) + ] postal_keys = {"street", "postal_code", "locality", "region", "country"} if postal_keys.intersection(mapped): postal = {key: mapped.get(key) for key in postal_keys if key in mapped} @@ -746,6 +865,8 @@ def _payload_from_mapped( "source_key": source_key, "locale": profile.configuration.get("locale"), "visibility": mapped.get("visibility"), + "source_format": profile.source_format, + "source_record_hash": (source_row or {}).get("__ldif_record_hash"), } } return ContactCreateRequest.model_validate(payload).model_dump( @@ -770,7 +891,19 @@ def _changed_fields(contact: Contact, mapped: dict[str, Any]) -> list[str]: if contact.postal_addresses: postal = contact.postal_addresses[0] current.update({key: getattr(postal, key) for key in ("street", "postal_code", "locality", "region", "country")}) - return sorted(key for key, value in mapped.items() if key != "visibility" and current.get(key) != value) + changed: list[str] = [] + for key, value in mapped.items(): + if key == "visibility": + continue + if key == "email" and isinstance(value, list): + current_value: Any = [item.email for item in contact.emails] + elif key == "phone" and isinstance(value, list): + current_value = [item.phone for item in contact.phones] + else: + current_value = current.get(key) + if current_value != value: + changed.append(key) + return sorted(changed) def _plan_effect( diff --git a/src/govoplan_addresses/backend/ldif.py b/src/govoplan_addresses/backend/ldif.py new file mode 100644 index 0000000..0a15968 --- /dev/null +++ b/src/govoplan_addresses/backend/ldif.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +import base64 +import binascii +import hashlib +from collections.abc import Iterable +from io import BytesIO +from typing import Any, Literal + + +LdifChangeRecordPolicy = Literal["reject", "ignore", "treat_add_as_entry"] + +MAX_LDIF_ATTRIBUTES = 500 +MAX_LDIF_VALUES_PER_ATTRIBUTE = 100 +MAX_LDIF_LOGICAL_LINE_BYTES = 1_000_000 + + +def parse_ldif_rows( + raw: bytes, + *, + max_entries: int, + change_record_policy: LdifChangeRecordPolicy = "reject", +) -> tuple[list[tuple[int, dict[str, Any]]], list[dict[str, Any]]]: + """Parse bounded LDIF entries without fetching URL or decoding binary values.""" + + rows: list[tuple[int, dict[str, Any]]] = [] + diagnostics: list[dict[str, Any]] = [] + record_lines: list[tuple[int, bytes]] = [] + + def finish_record() -> None: + if not record_lines: + return + row_number = record_lines[0][0] + row, record_diagnostics = _parse_record(record_lines) + diagnostics.extend(record_diagnostics) + record_lines.clear() + if not row: + return + if set(row) == {"version"} and _first(row.get("version")) == "1": + return + change_type = _first(row.get("changetype")).casefold() + if change_type: + if change_record_policy == "ignore": + diagnostics.append( + _diagnostic( + "warning", + "ldif_change_record_ignored", + f"LDIF change record {change_type!r} was ignored by profile policy.", + row_number=row_number, + field="changetype", + ) + ) + return + if change_record_policy != "treat_add_as_entry" or change_type != "add": + diagnostics.append( + _diagnostic( + "error", + "ldif_change_record_rejected", + f"LDIF change record {change_type!r} is not permitted by the profile policy.", + row_number=row_number, + field="changetype", + ) + ) + return + diagnostics.append( + _diagnostic( + "info", + "ldif_add_record_imported", + "LDIF add change record is treated as a static contact entry by profile policy.", + row_number=row_number, + field="changetype", + ) + ) + row.pop("changetype", None) + row["__ldif_record_hash"] = hashlib.sha256(_canonical_record(row)).hexdigest() + rows.append((row_number, row)) + if len(rows) > max_entries: + raise ValueError(f"LDIF exceeds the configured {max_entries}-entry limit.") + + for line_number, logical_line in _logical_lines(raw): + if not logical_line: + finish_record() + continue + if logical_line.startswith(b"#"): + continue + record_lines.append((line_number, logical_line)) + finish_record() + if not rows and not any(item["severity"] == "error" for item in diagnostics): + diagnostics.append(_diagnostic("warning", "ldif_no_entries", "No importable LDIF entries were found.")) + return rows, diagnostics + + +def _logical_lines(raw: bytes) -> Iterable[tuple[int, bytes]]: + current: bytearray | None = None + start_line = 0 + for line_number, physical_with_ending in enumerate(BytesIO(raw), start=1): + physical = physical_with_ending.rstrip(b"\r\n") + if physical.startswith(b" "): + if current is None: + yield line_number, b"!invalid-fold-without-preceding-line" + continue + current.extend(physical[1:]) + if len(current) > MAX_LDIF_LOGICAL_LINE_BYTES: + raise ValueError(f"LDIF logical line starting at {start_line} exceeds the size limit.") + continue + if current is not None: + yield start_line, bytes(current) + current = bytearray(physical) + start_line = line_number + if len(current) > MAX_LDIF_LOGICAL_LINE_BYTES: + raise ValueError(f"LDIF logical line {line_number} exceeds the size limit.") + if current is not None: + yield start_line, bytes(current) + + +def _parse_record( + lines: list[tuple[int, bytes]], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + row: dict[str, list[str]] = {} + diagnostics: list[dict[str, Any]] = [] + for line_number, line in lines: + if line == b"!invalid-fold-without-preceding-line": + diagnostics.append( + _diagnostic( + "error", + "ldif_invalid_fold", + "LDIF continuation line has no preceding attribute.", + row_number=line_number, + ) + ) + continue + if b":" not in line: + diagnostics.append( + _diagnostic( + "error", + "ldif_invalid_line", + "LDIF line is missing the attribute separator.", + row_number=line_number, + ) + ) + continue + if line == b"-": + # Attribute-operation separators are meaningful only inside change + # records, whose enclosing policy is evaluated after the record. + continue + raw_name, raw_value = line.split(b":", 1) + try: + name_parts = [part.strip().casefold() for part in raw_name.decode("ascii").split(";")] + name = name_parts[0] + except UnicodeDecodeError: + name_parts = [] + name = "" + if not name or any(character.isspace() for character in name): + diagnostics.append( + _diagnostic( + "error", + "ldif_invalid_attribute", + "LDIF attribute name is invalid.", + row_number=line_number, + ) + ) + continue + if "binary" in name_parts[1:]: + diagnostics.append( + _diagnostic( + "warning", + "ldif_binary_value_ignored", + f"Binary LDIF attribute {name!r} was ignored; binary data is never projected into contacts.", + row_number=line_number, + field=name, + ) + ) + continue + if name not in row and len(row) >= MAX_LDIF_ATTRIBUTES: + diagnostics.append( + _diagnostic( + "error", + "ldif_too_many_attributes", + f"LDIF entry exceeds the {MAX_LDIF_ATTRIBUTES}-attribute limit.", + row_number=line_number, + ) + ) + continue + value, value_diagnostic = _decode_value(raw_value, attribute=name, line_number=line_number) + if value_diagnostic is not None: + diagnostics.append(value_diagnostic) + if value is None: + continue + values = row.setdefault(name, []) + if len(values) >= MAX_LDIF_VALUES_PER_ATTRIBUTE: + diagnostics.append( + _diagnostic( + "error", + "ldif_too_many_values", + f"LDIF attribute {name!r} exceeds the value limit.", + row_number=line_number, + field=name, + ) + ) + continue + values.append(value) + return row, diagnostics + + +def _decode_value( + raw_value: bytes, + *, + attribute: str, + line_number: int, +) -> tuple[str | None, dict[str, Any] | None]: + if raw_value.startswith(b":"): + encoded = raw_value[1:].lstrip(b" ") + try: + decoded = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError): + return None, _diagnostic( + "error", + "ldif_invalid_base64", + f"LDIF attribute {attribute!r} contains invalid base64.", + row_number=line_number, + field=attribute, + ) + try: + return decoded.decode("utf-8"), None + except UnicodeDecodeError: + return None, _diagnostic( + "warning", + "ldif_binary_value_ignored", + f"Binary LDIF attribute {attribute!r} was ignored; binary data is never projected into contacts.", + row_number=line_number, + field=attribute, + ) + if raw_value.startswith(b"<"): + return None, _diagnostic( + "warning", + "ldif_url_value_ignored", + f"External LDIF URL value for {attribute!r} was ignored; imports never fetch referenced content.", + row_number=line_number, + field=attribute, + ) + value_bytes = raw_value[1:] if raw_value.startswith(b" ") else raw_value + try: + return value_bytes.decode("utf-8"), None + except UnicodeDecodeError: + return None, _diagnostic( + "error", + "ldif_invalid_utf8", + f"LDIF attribute {attribute!r} is not valid UTF-8.", + row_number=line_number, + field=attribute, + ) + + +def _canonical_record(row: dict[str, Any]) -> bytes: + lines = [] + for name in sorted(key for key in row if not key.startswith("__")): + values = row[name] if isinstance(row[name], list) else [row[name]] + lines.extend(f"{name}:{value}" for value in values) + return "\n".join(lines).encode("utf-8") + + +def _first(value: object) -> str: + if isinstance(value, list): + return str(value[0]) if value else "" + return str(value or "") + + +def _diagnostic( + severity: str, + code: str, + message: str, + *, + row_number: int | None = None, + field: str | None = None, +) -> dict[str, Any]: + return { + "severity": severity, + "code": code, + "message": message, + "row_number": row_number, + "field": field, + "details": {}, + } + + +__all__ = ["LdifChangeRecordPolicy", "parse_ldif_rows"] diff --git a/src/govoplan_addresses/backend/manifest.py b/src/govoplan_addresses/backend/manifest.py index 8fda87e..667e070 100644 --- a/src/govoplan_addresses/backend/manifest.py +++ b/src/govoplan_addresses/backend/manifest.py @@ -440,17 +440,19 @@ manifest = ModuleManifest( ), DocumentationTopic( id="addresses.tabular-imports", - title="CSV and XLSX contact imports", - summary="Preview and apply reusable, versioned contact mappings without silent row loss.", + title="CSV, XLSX, and LDIF contact imports", + summary="Preview and apply reusable, versioned contact mappings without silent row or entry loss.", body=( - "CSV and XLSX files can be mapped with scoped, reusable profile versions. Each preview validates headers, " - "encodings, source keys, duplicates, blank values, workbook limits, and contact identity before any mutation. " + "CSV, XLSX, and LDIF files can be mapped with scoped, reusable profile versions. Each preview validates headers or attributes, " + "encodings, source keys, duplicates, blank values, format limits, and contact identity before any mutation. " "The reviewed input hash and plan hash are retained with row-level effects and diagnostics. Apply is idempotent, " "rejects contacts changed after preview, and records sufficient evidence for a guarded rollback. A persisted run " "can be reopened with its run link after navigation or reload; previewed, applied, rolled-back, expired, and " "unavailable states remain explicit. Both apply and rollback submit the reviewed plan hash. Missing, expired, " "hidden, and cross-tenant runs disclose no source payload. XLSX formulas, macros, and legacy workbook formats " - "are never executed or imported." + "are never executed or imported. LDIF folded lines, UTF-8 and base64 text, repeated attributes, and comments are parsed; " + "binary and URL values are never projected or fetched. Change records default to rejected diagnostics and may only be ignored " + "or treat add records as static entries through an explicit profile policy." ), layer="configured", documentation_types=("admin", "user"), diff --git a/tests/test_ldif_imports.py b/tests/test_ldif_imports.py new file mode 100644 index 0000000..8f66ca0 --- /dev/null +++ b/tests/test_ldif_imports.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import base64 +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_addresses.backend.db.models import AddressBook, Contact +from govoplan_addresses.backend.import_schemas import ( + AddressImportConfiguration, + AddressImportPreviewRequest, + AddressImportProfileCreateRequest, +) +from govoplan_addresses.backend.imports import ( + apply_address_import, + create_import_profile, + import_run_payload, + preview_address_import, +) +from govoplan_addresses.backend.ldif import parse_ldif_rows +from govoplan_core.db.base import Base + + +class Principal: + account_id = "account-1" + group_ids = frozenset() + + @property + def tenant_id(self) -> str: + return "tenant-1" + + def has(self, scope: str) -> bool: + return scope in { + "addresses:address_book:read", + "addresses:address_book:write", + "addresses:contact:read", + "addresses:contact:write", + } + + +def encoded(value: bytes | str) -> str: + raw = value.encode() if isinstance(value, str) else value + return base64.b64encode(raw).decode() + + +class AddressLdifImportTests(unittest.TestCase): + def setUp(self) -> None: + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + self.session = sessionmaker(bind=engine, expire_on_commit=False)() + self.principal = Principal() + self.book = AddressBook( + tenant_id="tenant-1", + scope_type="tenant", + scope_id="tenant-1", + name="Imported contacts", + source_kind="local", + read_only=False, + ) + self.session.add(self.book) + + def create_profile(self, **configuration_overrides): + configuration = AddressImportConfiguration( + field_mappings={ + "source_key": "dn", + "display_name": "cn", + "given_name": "givenName", + "family_name": "sn", + "email": "mail", + "phone": "telephoneNumber", + "organization": "o", + }, + **configuration_overrides, + ) + profile = create_import_profile( + self.session, + self.principal, + AddressImportProfileCreateRequest( + scope_type="tenant", + name="Directory export", + source_format="ldif", + configuration=configuration, + ), + ) + self.session.flush() + return profile + + def test_parser_unfolds_and_decodes_text_without_projecting_binary_or_urls(self) -> None: + rows, diagnostics = parse_ldif_rows( + b"version: 1\n\n" + b"# exported contact\n" + b"dn: uid=ada,ou=people,dc=example,dc=test\n" + b"cn:: QWRhIExvdmVsYWNl\n" + b"sn: Love\n" + b" lace\n" + b"mail: ada@example.test\n" + b"mail: ada.work@example.test\n" + b"jpegPhoto:: /9j/\n" + b"seeAlso:< https://example.test/contact/ada\n", + max_entries=10, + ) + + self.assertEqual(1, len(rows)) + self.assertEqual(["Ada Lovelace"], rows[0][1]["cn"]) + self.assertEqual(["Lovelace"], rows[0][1]["sn"]) + self.assertEqual(["ada@example.test", "ada.work@example.test"], rows[0][1]["mail"]) + self.assertNotIn("jpegphoto", rows[0][1]) + self.assertNotIn("seealso", rows[0][1]) + self.assertEqual( + {"ldif_binary_value_ignored", "ldif_url_value_ignored"}, + {item["code"] for item in diagnostics}, + ) + + def test_preview_apply_and_repeat_preserve_multivalue_provenance(self) -> None: + profile = self.create_profile(default_tags=["ldif"]) + source = ( + "dn: uid=ada,ou=people,dc=example,dc=test\n" + "cn: Ada Lovelace\n" + "givenName: Ada\n" + "sn: Lovelace\n" + "mail: ada@example.test\n" + "mail: ada.work@example.test\n" + "telephoneNumber: +49 30 123\n" + "o: Analysis Office\n" + ) + request = AddressImportPreviewRequest( + profile_id=profile.id, + filename="contacts.ldif", + content_base64=encoded(source), + ) + run = preview_address_import(self.session, self.principal, self.book.id, request) + self.assertEqual(1, run.statistics["create"]) + self.assertFalse([item for item in run.diagnostics if item["severity"] == "error"]) + + apply_address_import(self.session, self.principal, run.id, expected_plan_hash=run.plan_hash) + contact = self.session.query(Contact).one() + self.assertEqual(["ada@example.test", "ada.work@example.test"], [item.email for item in contact.emails]) + self.assertEqual("ldif", contact.source_kind) + self.assertEqual("ldif", contact.provenance["import"]["source_format"]) + self.assertEqual(64, len(contact.provenance["import"]["source_record_hash"])) + self.assertNotIn(source, repr(import_run_payload(run))) + + repeated = preview_address_import(self.session, self.principal, self.book.id, request) + self.assertEqual(1, repeated.statistics["unchanged"]) + apply_address_import(self.session, self.principal, repeated.id, expected_plan_hash=repeated.plan_hash) + self.assertEqual(1, self.session.query(Contact).count()) + + def test_change_records_are_rejected_by_default_and_add_requires_explicit_policy(self) -> None: + source = ( + "dn: uid=ada,ou=people,dc=example,dc=test\n" + "changetype: add\n" + "cn: Ada Lovelace\n" + "mail: ada@example.test\n" + ) + rejected_profile = self.create_profile() + rejected = preview_address_import( + self.session, + self.principal, + self.book.id, + AddressImportPreviewRequest( + profile_id=rejected_profile.id, + filename="changes.ldif", + content_base64=encoded(source), + ), + ) + self.assertEqual(0, rejected.row_count) + self.assertIn("ldif_change_record_rejected", {item["code"] for item in rejected.diagnostics}) + self.assertFalse(import_run_payload(rejected)["can_apply"]) + + allowed_profile = self.create_profile(ldif_change_record_policy="treat_add_as_entry") + allowed = preview_address_import( + self.session, + self.principal, + self.book.id, + AddressImportPreviewRequest( + profile_id=allowed_profile.id, + filename="changes.ldif", + content_base64=encoded(source), + ), + ) + self.assertEqual(1, allowed.statistics["create"]) + self.assertIn("ldif_add_record_imported", {item["code"] for item in allowed.diagnostics}) + + def test_invalid_base64_is_a_correction_diagnostic_and_blocks_apply(self) -> None: + profile = self.create_profile() + run = preview_address_import( + self.session, + self.principal, + self.book.id, + AddressImportPreviewRequest( + profile_id=profile.id, + filename="broken.ldif", + content_base64=encoded( + "dn: uid=ada,dc=example,dc=test\n" + "cn:: this-is-not-base64!\n" + "mail: ada@example.test\n" + ), + ), + ) + self.assertIn("ldif_invalid_base64", {item["code"] for item in run.diagnostics}) + self.assertFalse(import_run_payload(run)["can_apply"]) + with self.assertRaisesRegex(ValueError, "error diagnostics"): + apply_address_import(self.session, self.principal, run.id, expected_plan_hash=run.plan_hash) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/src/api/addresses.ts b/webui/src/api/addresses.ts index 06c7262..900398f 100644 --- a/webui/src/api/addresses.ts +++ b/webui/src/api/addresses.ts @@ -494,6 +494,7 @@ export type AddressImportConfiguration = { duplicate_source_key_policy: "reject" | "first" | "last"; existing_contact_policy: "update" | "ignore" | "reject"; blank_value_policy: "ignore" | "clear" | "reject"; + ldif_change_record_policy: "reject" | "ignore" | "treat_add_as_entry"; locale?: string | null; default_tags: string[]; max_rows: number; @@ -508,7 +509,7 @@ export type AddressImportProfile = { scope_id?: string | null; name: string; description?: string | null; - source_format: "csv" | "xlsx"; + source_format: "csv" | "xlsx" | "ldif"; configuration: AddressImportConfiguration; is_current: boolean; created_at: string; @@ -1068,7 +1069,7 @@ export function createAddressImportProfile( scope_id?: string | null; name: string; description?: string | null; - source_format: "csv" | "xlsx"; + source_format: "csv" | "xlsx" | "ldif"; configuration: AddressImportConfiguration; } ): Promise { diff --git a/webui/src/features/addressbook/AddressBookPage.tsx b/webui/src/features/addressbook/AddressBookPage.tsx index cb80959..059f050 100644 --- a/webui/src/features/addressbook/AddressBookPage.tsx +++ b/webui/src/features/addressbook/AddressBookPage.tsx @@ -237,7 +237,7 @@ type ImportMode = "vcard" | "tabular"; type ImportProfileFormState = { name: string; - source_format: "csv" | "xlsx"; + source_format: "csv" | "xlsx" | "ldif"; delimiter: "," | ";" | "\t" | "|"; encoding: "utf-8" | "utf-8-sig" | "cp1252" | "latin-1"; header_row: string; @@ -245,6 +245,7 @@ type ImportProfileFormState = { duplicate_source_key_policy: "reject" | "first" | "last"; existing_contact_policy: "update" | "ignore" | "reject"; blank_value_policy: "ignore" | "clear" | "reject"; + ldif_change_record_policy: "reject" | "ignore" | "treat_add_as_entry"; locale: string; default_tags: string; max_rows: string; @@ -398,6 +399,39 @@ const EMPTY_LDAP_FORM: LdapFormState = { attribute_map: DEFAULT_LDAP_ATTRIBUTE_MAP }; +const DEFAULT_TABULAR_FIELD_MAPPINGS: Record = { + source_key: "id", + display_name: "display_name", + given_name: "given_name", + family_name: "family_name", + organization: "organization", + role_title: "role_title", + email: "email", + phone: "phone", + street: "street", + postal_code: "postal_code", + locality: "locality", + region: "region", + country: "country", + tags: "tags" +}; + +const DEFAULT_LDIF_FIELD_MAPPINGS: Record = { + source_key: "dn", + display_name: "cn", + given_name: "givenname", + family_name: "sn", + organization: "o", + role_title: "title", + email: "mail", + phone: "telephonenumber", + street: "street", + postal_code: "postalcode", + locality: "l", + region: "st", + country: "c" +}; + const EMPTY_IMPORT_PROFILE_FORM: ImportProfileFormState = { name: "", source_format: "csv", @@ -408,25 +442,11 @@ const EMPTY_IMPORT_PROFILE_FORM: ImportProfileFormState = { duplicate_source_key_policy: "reject", existing_contact_policy: "update", blank_value_policy: "ignore", + ldif_change_record_policy: "reject", locale: "", default_tags: "", max_rows: "10000", - field_mappings: { - source_key: "id", - display_name: "display_name", - given_name: "given_name", - family_name: "family_name", - organization: "organization", - role_title: "role_title", - email: "email", - phone: "phone", - street: "street", - postal_code: "postal_code", - locality: "locality", - region: "region", - country: "country", - tags: "tags" - } + field_mappings: DEFAULT_TABULAR_FIELD_MAPPINGS }; const IMPORT_MAPPING_FIELDS = [ @@ -2384,6 +2404,7 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) duplicate_source_key_policy: importProfileForm.duplicate_source_key_policy, existing_contact_policy: importProfileForm.existing_contact_policy, blank_value_policy: importProfileForm.blank_value_policy, + ldif_change_record_policy: importProfileForm.ldif_change_record_policy, locale: importProfileForm.locale.trim() || null, default_tags: importProfileForm.default_tags.split(",").map((tag) => tag.trim()).filter(Boolean), max_rows: Number(importProfileForm.max_rows) || 10000 @@ -2438,6 +2459,7 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) duplicate_source_key_policy: config.duplicate_source_key_policy, existing_contact_policy: config.existing_contact_policy, blank_value_policy: config.blank_value_policy, + ldif_change_record_policy: config.ldif_change_record_policy ?? "reject", locale: config.locale ?? "", default_tags: config.default_tags.join(", "), max_rows: String(config.max_rows), @@ -3798,7 +3820,7 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) role="group" size="equal" ariaLabel="Contact import format" - options={[{ id: "vcard", label: "vCard" }, { id: "tabular", label: "CSV / XLSX" }]} + options={[{ id: "vcard", label: "vCard" }, { id: "tabular", label: "Mapped file" }]} value={importMode} onChange={(mode) => { setImportMode(mode); clearRetainedImportRun(); }} /> @@ -3861,9 +3883,18 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) setImportProfileForm((current) => ({ ...current, name: event.target.value }))} /> - setImportProfileForm((current) => ({ + ...current, + source_format: event.target.value as ImportProfileFormState["source_format"], + field_mappings: event.target.value === "ldif" + ? DEFAULT_LDIF_FIELD_MAPPINGS + : current.source_format === "ldif" + ? DEFAULT_TABULAR_FIELD_MAPPINGS + : current.field_mappings + }))}> + {importProfileForm.source_format === "csv" && <> @@ -3879,7 +3910,15 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) } {importProfileForm.source_format === "xlsx" && setImportProfileForm((current) => ({ ...current, sheet_name: event.target.value }))} placeholder="First sheet" />} - setImportProfileForm((current) => ({ ...current, header_row: event.target.value }))} /> + {importProfileForm.source_format !== "ldif" && setImportProfileForm((current) => ({ ...current, header_row: event.target.value }))} />} + {importProfileForm.source_format === "ldif" && + + + } setImportProfileForm((current) => ({ ...current, field_mappings: { ...current.field_mappings, [target]: event.target.value } }))} - placeholder="Source column" + placeholder={importProfileForm.source_format === "ldif" ? "LDIF attribute" : "Source column"} /> )} @@ -3914,7 +3953,7 @@ export default function AddressBookPage({ settings, auth, onAuthChange }: Props) { setImportFile(event.target.files?.[0] ?? null); clearRetainedImportRun(); }} />