feat: add governed LDIF contact imports

src/govoplan_addresses/backend/import_schemas.py

@@ -6,7 +6,7 @@ from typing import Any, Literal
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 
-AddressImportFormat = Literal["csv", "xlsx"]
+AddressImportFormat = Literal["csv", "xlsx", "ldif"]
 AddressImportScope = Literal["user", "group", "tenant", "system"]
 
 IMPORT_TARGET_FIELDS = frozenset(
@@ -41,6 +41,7 @@ class AddressImportConfiguration(BaseModel):
     duplicate_source_key_policy: Literal["reject", "first", "last"] = "reject"
     existing_contact_policy: Literal["update", "ignore", "reject"] = "update"
     blank_value_policy: Literal["ignore", "clear", "reject"] = "ignore"
+    ldif_change_record_policy: Literal["reject", "ignore", "treat_add_as_entry"] = "reject"
     locale: str | None = Field(default=None, max_length=35)
     default_tags: list[str] = Field(default_factory=list, max_length=100)
     max_rows: int = Field(default=10_000, ge=1, le=10_000)

src/govoplan_addresses/backend/imports.py

@@ -24,6 +24,7 @@ from govoplan_addresses.backend.import_schemas import (
     AddressImportProfileUpdateRequest,
     AddressImportRollbackRequest,
 )
+from govoplan_addresses.backend.ldif import parse_ldif_rows
 from govoplan_addresses.backend.schemas import (
     ContactCreateRequest,
     ContactEmailPayload,
@@ -199,6 +200,7 @@ def preview_address_import(
         input_hash=input_hash,
         rows=rows,
         config=config,
+        source_format=profile.source_format,
     )
     diagnostics = [*parse_diagnostics, *map_diagnostics]
     statistics = dict(Counter(item["action"] for item in plan_data))
@@ -501,13 +503,24 @@ def _parse_rows(
     filename: str,
     source_format: str,
     config: AddressImportConfiguration,
-) -> tuple[list[tuple[int, dict[str, str]]], list[dict[str, Any]]]:
+) -> tuple[list[tuple[int, dict[str, Any]]], list[dict[str, Any]]]:
     if source_format == "csv":
         return _parse_csv(raw, config=config)
     if source_format == "xlsx":
         if not filename.casefold().endswith(".xlsx"):
             raise AddressBookError("XLSX imports require an .xlsx file; macros and legacy workbooks are not accepted.")
         return _parse_xlsx(raw, config=config)
+    if source_format == "ldif":
+        if not filename.casefold().endswith((".ldif", ".ldi")):
+            raise AddressBookError("LDIF imports require an .ldif or .ldi file.")
+        try:
+            return parse_ldif_rows(
+                raw,
+                max_entries=config.max_rows,
+                change_record_policy=config.ldif_change_record_policy,
+            )
+        except ValueError as exc:
+            raise AddressBookError(str(exc)) from exc
     raise AddressBookError(f"Unsupported address import format: {source_format!r}.")
 
 
@@ -600,32 +613,81 @@ def _row_dict(headers: list[str], values: list[Any]) -> dict[str, str]:
     }
 
 
+def _column_key(value: object, *, casefold: bool) -> str:
+    column = str(value).strip()
+    return column.casefold() if casefold else column
+
+
+def _row_value(row: dict[str, Any], column: str, *, casefold: bool) -> object:
+    if column in row:
+        return row[column]
+    if not casefold:
+        return ""
+    expected = column.casefold()
+    for key, value in row.items():
+        if key.casefold() == expected:
+            return value
+    return ""
+
+
+def _import_values(value: object) -> list[str]:
+    raw_values = value if isinstance(value, list) else [value]
+    return [str(item).strip() for item in raw_values if item is not None and str(item).strip()]
+
+
+def _first_import_value(value: object) -> str:
+    values = _import_values(value)
+    return values[0] if values else ""
+
+
 def _plan_rows(
     session: Session,
     *,
     book_id: str,
     profile: AddressImportProfile,
     input_hash: str,
-    rows: list[tuple[int, dict[str, str]]],
+    rows: list[tuple[int, dict[str, Any]]],
     config: AddressImportConfiguration,
+    source_format: str,
 ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     diagnostics: list[dict[str, Any]] = []
     plan: list[dict[str, Any]] = []
-    headers = set(rows[0][1]) if rows else set()
-    referenced_columns = set(config.field_mappings.values())
+    casefold_columns = source_format == "ldif"
+    headers = {
+        _column_key(column, casefold=casefold_columns)
+        for column in (rows[0][1] if rows else set())
+        if not str(column).startswith("__")
+    }
+    referenced_columns = {
+        _column_key(column, casefold=casefold_columns)
+        for column in config.field_mappings.values()
+    }
+    key_column = config.source_key_column or config.field_mappings["source_key"]
+    normalized_key_column = _column_key(key_column, casefold=casefold_columns)
     if config.source_key_column:
-        referenced_columns.add(config.source_key_column)
+        referenced_columns.add(_column_key(config.source_key_column, casefold=casefold_columns))
     missing_columns = sorted(referenced_columns.difference(headers))
     for column in missing_columns:
-        diagnostics.append(_diagnostic("error", "missing_column", f'Configured column "{column}" is missing.', field=column))
-    if missing_columns:
+        optional_ldif_attribute = casefold_columns and column != normalized_key_column
+        diagnostics.append(
+            _diagnostic(
+                "warning" if optional_ldif_attribute else "error",
+                "missing_attribute" if optional_ldif_attribute else "missing_column",
+                (
+                    f'Configured LDIF attribute "{column}" is absent from this file.'
+                    if optional_ldif_attribute
+                    else f'Configured column "{column}" is missing.'
+                ),
+                field=column,
+            )
+        )
+    if normalized_key_column in missing_columns or (missing_columns and not casefold_columns):
         return [], diagnostics
 
-    key_column = config.source_key_column or config.field_mappings["source_key"]
-    keyed_rows: list[tuple[int, dict[str, str], str]] = []
+    keyed_rows: list[tuple[int, dict[str, Any], str]] = []
     key_counts: Counter[str] = Counter()
     for row_number, row in rows:
-        key = row.get(key_column, "").strip()
+        key = _first_import_value(_row_value(row, key_column, casefold=casefold_columns)).strip()
         if not key:
             diagnostics.append(_diagnostic("error", "missing_source_key", "Stable source key is blank.", row_number=row_number, field=key_column))
             plan.append(_plan_effect(row_number, "conflict", source_key=None, message="Stable source key is blank."))
@@ -651,11 +713,23 @@ def _plan_rows(
                 plan.append(_plan_effect(row_number, "ignored", source_key=key, message="Duplicate row ignored by profile policy."))
                 continue
 
-        mapped, row_diagnostics = _mapped_fields(row_number, row, config=config)
+        mapped, row_diagnostics = _mapped_fields(
+            row_number,
+            row,
+            config=config,
+            casefold_columns=casefold_columns,
+        )
         diagnostics.extend(row_diagnostics)
         source_ref = f"import:{profile.profile_key}:{key}"
         existing = _contact_by_source_ref(session, book_id, source_ref)
-        payload = _payload_from_mapped(mapped, profile=profile, input_hash=input_hash, row_number=row_number, source_key=key)
+        payload = _payload_from_mapped(
+            mapped,
+            profile=profile,
+            input_hash=input_hash,
+            row_number=row_number,
+            source_key=key,
+            source_row=row,
+        )
         display_name = payload.get("display_name") or payload.get("email") or key
         if any(item["severity"] == "error" for item in row_diagnostics):
             plan.append(_plan_effect(row_number, "conflict", source_key=key, display_name=display_name, source_ref=source_ref, payload=payload, message="Row validation failed."))
@@ -689,23 +763,46 @@ def _plan_rows(
 
 def _mapped_fields(
     row_number: int,
-    row: dict[str, str],
+    row: dict[str, Any],
     *,
     config: AddressImportConfiguration,
+    casefold_columns: bool = False,
 ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
     mapped: dict[str, Any] = {}
     diagnostics: list[dict[str, Any]] = []
     for target, column in config.field_mappings.items():
         if target == "source_key":
             continue
-        value = row.get(column, "").strip()
-        if not value:
+        values = _import_values(_row_value(row, column, casefold=casefold_columns))
+        if not values:
             if config.blank_value_policy == "reject":
                 diagnostics.append(_diagnostic("error", "blank_value", f'Column "{column}" is blank.', row_number=row_number, field=target))
             elif config.blank_value_policy == "clear":
                 mapped[target] = [] if target == "tags" else None
             continue
-        mapped[target] = [item.strip() for item in value.split(",") if item.strip()] if target == "tags" else value
+        if target == "tags":
+            mapped[target] = list(
+                dict.fromkeys(
+                    item.strip()
+                    for value in values
+                    for item in value.split(",")
+                    if item.strip()
+                )
+            )
+        elif target in {"email", "phone"}:
+            mapped[target] = values if len(values) > 1 else values[0]
+        else:
+            mapped[target] = values[0]
+            if len(values) > 1:
+                diagnostics.append(
+                    _diagnostic(
+                        "warning",
+                        "multiple_values_truncated",
+                        f'Attribute "{column}" has multiple values; only the first maps to {target!r}.',
+                        row_number=row_number,
+                        field=target,
+                    )
+                )
     if config.default_tags:
         mapped["tags"] = list(dict.fromkeys([*(mapped.get("tags") or []), *config.default_tags]))
     if not any(mapped.get(name) for name in ("display_name", "given_name", "family_name", "email", "organization")):
@@ -720,8 +817,22 @@ def _payload_from_mapped(
     input_hash: str,
     row_number: int,
     source_key: str,
+    source_row: dict[str, Any] | None = None,
 ) -> dict[str, Any]:
-    display_name = mapped.get("display_name") or " ".join(filter(None, [mapped.get("given_name"), mapped.get("family_name")])) or mapped.get("email") or mapped.get("organization")
+    display_name = (
+        _first_import_value(mapped.get("display_name"))
+        or " ".join(
+            filter(
+                None,
+                [
+                    _first_import_value(mapped.get("given_name")),
+                    _first_import_value(mapped.get("family_name")),
+                ],
+            )
+        )
+        or _first_import_value(mapped.get("email"))
+        or _first_import_value(mapped.get("organization"))
+    )
     payload: dict[str, Any] = {
         key: mapped.get(key)
         for key in ("given_name", "family_name", "organization", "role_title", "note", "tags")
@@ -729,9 +840,17 @@ def _payload_from_mapped(
     }
     payload["display_name"] = display_name
     if "email" in mapped:
-        payload["emails"] = [] if mapped["email"] is None else [ContactEmailPayload(email=mapped["email"], is_primary=True).model_dump(mode="json")]
+        email_values = _import_values(mapped["email"])
+        payload["emails"] = [
+            ContactEmailPayload(email=value, is_primary=index == 0).model_dump(mode="json")
+            for index, value in enumerate(email_values)
+        ]
     if "phone" in mapped:
-        payload["phones"] = [] if mapped["phone"] is None else [ContactPhonePayload(phone=mapped["phone"], is_primary=True).model_dump(mode="json")]
+        phone_values = _import_values(mapped["phone"])
+        payload["phones"] = [
+            ContactPhonePayload(phone=value, is_primary=index == 0).model_dump(mode="json")
+            for index, value in enumerate(phone_values)
+        ]
     postal_keys = {"street", "postal_code", "locality", "region", "country"}
     if postal_keys.intersection(mapped):
         postal = {key: mapped.get(key) for key in postal_keys if key in mapped}
@@ -746,6 +865,8 @@ def _payload_from_mapped(
             "source_key": source_key,
             "locale": profile.configuration.get("locale"),
             "visibility": mapped.get("visibility"),
+            "source_format": profile.source_format,
+            "source_record_hash": (source_row or {}).get("__ldif_record_hash"),
         }
     }
     return ContactCreateRequest.model_validate(payload).model_dump(
@@ -770,7 +891,19 @@ def _changed_fields(contact: Contact, mapped: dict[str, Any]) -> list[str]:
     if contact.postal_addresses:
         postal = contact.postal_addresses[0]
         current.update({key: getattr(postal, key) for key in ("street", "postal_code", "locality", "region", "country")})
-    return sorted(key for key, value in mapped.items() if key != "visibility" and current.get(key) != value)
+    changed: list[str] = []
+    for key, value in mapped.items():
+        if key == "visibility":
+            continue
+        if key == "email" and isinstance(value, list):
+            current_value: Any = [item.email for item in contact.emails]
+        elif key == "phone" and isinstance(value, list):
+            current_value = [item.phone for item in contact.phones]
+        else:
+            current_value = current.get(key)
+        if current_value != value:
+            changed.append(key)
+    return sorted(changed)
 
 
 def _plan_effect(

src/govoplan_addresses/backend/ldif.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+from collections.abc import Iterable
+from io import BytesIO
+from typing import Any, Literal
+
+
+LdifChangeRecordPolicy = Literal["reject", "ignore", "treat_add_as_entry"]
+
+MAX_LDIF_ATTRIBUTES = 500
+MAX_LDIF_VALUES_PER_ATTRIBUTE = 100
+MAX_LDIF_LOGICAL_LINE_BYTES = 1_000_000
+
+
+def parse_ldif_rows(
+    raw: bytes,
+    *,
+    max_entries: int,
+    change_record_policy: LdifChangeRecordPolicy = "reject",
+) -> tuple[list[tuple[int, dict[str, Any]]], list[dict[str, Any]]]:
+    """Parse bounded LDIF entries without fetching URL or decoding binary values."""
+
+    rows: list[tuple[int, dict[str, Any]]] = []
+    diagnostics: list[dict[str, Any]] = []
+    record_lines: list[tuple[int, bytes]] = []
+
+    def finish_record() -> None:
+        if not record_lines:
+            return
+        row_number = record_lines[0][0]
+        row, record_diagnostics = _parse_record(record_lines)
+        diagnostics.extend(record_diagnostics)
+        record_lines.clear()
+        if not row:
+            return
+        if set(row) == {"version"} and _first(row.get("version")) == "1":
+            return
+        change_type = _first(row.get("changetype")).casefold()
+        if change_type:
+            if change_record_policy == "ignore":
+                diagnostics.append(
+                    _diagnostic(
+                        "warning",
+                        "ldif_change_record_ignored",
+                        f"LDIF change record {change_type!r} was ignored by profile policy.",
+                        row_number=row_number,
+                        field="changetype",
+                    )
+                )
+                return
+            if change_record_policy != "treat_add_as_entry" or change_type != "add":
+                diagnostics.append(
+                    _diagnostic(
+                        "error",
+                        "ldif_change_record_rejected",
+                        f"LDIF change record {change_type!r} is not permitted by the profile policy.",
+                        row_number=row_number,
+                        field="changetype",
+                    )
+                )
+                return
+            diagnostics.append(
+                _diagnostic(
+                    "info",
+                    "ldif_add_record_imported",
+                    "LDIF add change record is treated as a static contact entry by profile policy.",
+                    row_number=row_number,
+                    field="changetype",
+                )
+            )
+            row.pop("changetype", None)
+        row["__ldif_record_hash"] = hashlib.sha256(_canonical_record(row)).hexdigest()
+        rows.append((row_number, row))
+        if len(rows) > max_entries:
+            raise ValueError(f"LDIF exceeds the configured {max_entries}-entry limit.")
+
+    for line_number, logical_line in _logical_lines(raw):
+        if not logical_line:
+            finish_record()
+            continue
+        if logical_line.startswith(b"#"):
+            continue
+        record_lines.append((line_number, logical_line))
+    finish_record()
+    if not rows and not any(item["severity"] == "error" for item in diagnostics):
+        diagnostics.append(_diagnostic("warning", "ldif_no_entries", "No importable LDIF entries were found."))
+    return rows, diagnostics
+
+
+def _logical_lines(raw: bytes) -> Iterable[tuple[int, bytes]]:
+    current: bytearray | None = None
+    start_line = 0
+    for line_number, physical_with_ending in enumerate(BytesIO(raw), start=1):
+        physical = physical_with_ending.rstrip(b"\r\n")
+        if physical.startswith(b" "):
+            if current is None:
+                yield line_number, b"!invalid-fold-without-preceding-line"
+                continue
+            current.extend(physical[1:])
+            if len(current) > MAX_LDIF_LOGICAL_LINE_BYTES:
+                raise ValueError(f"LDIF logical line starting at {start_line} exceeds the size limit.")
+            continue
+        if current is not None:
+            yield start_line, bytes(current)
+        current = bytearray(physical)
+        start_line = line_number
+        if len(current) > MAX_LDIF_LOGICAL_LINE_BYTES:
+            raise ValueError(f"LDIF logical line {line_number} exceeds the size limit.")
+    if current is not None:
+        yield start_line, bytes(current)
+
+
+def _parse_record(
+    lines: list[tuple[int, bytes]],
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    row: dict[str, list[str]] = {}
+    diagnostics: list[dict[str, Any]] = []
+    for line_number, line in lines:
+        if line == b"!invalid-fold-without-preceding-line":
+            diagnostics.append(
+                _diagnostic(
+                    "error",
+                    "ldif_invalid_fold",
+                    "LDIF continuation line has no preceding attribute.",
+                    row_number=line_number,
+                )
+            )
+            continue
+        if b":" not in line:
+            diagnostics.append(
+                _diagnostic(
+                    "error",
+                    "ldif_invalid_line",
+                    "LDIF line is missing the attribute separator.",
+                    row_number=line_number,
+                )
+            )
+            continue
+        if line == b"-":
+            # Attribute-operation separators are meaningful only inside change
+            # records, whose enclosing policy is evaluated after the record.
+            continue
+        raw_name, raw_value = line.split(b":", 1)
+        try:
+            name_parts = [part.strip().casefold() for part in raw_name.decode("ascii").split(";")]
+            name = name_parts[0]
+        except UnicodeDecodeError:
+            name_parts = []
+            name = ""
+        if not name or any(character.isspace() for character in name):
+            diagnostics.append(
+                _diagnostic(
+                    "error",
+                    "ldif_invalid_attribute",
+                    "LDIF attribute name is invalid.",
+                    row_number=line_number,
+                )
+            )
+            continue
+        if "binary" in name_parts[1:]:
+            diagnostics.append(
+                _diagnostic(
+                    "warning",
+                    "ldif_binary_value_ignored",
+                    f"Binary LDIF attribute {name!r} was ignored; binary data is never projected into contacts.",
+                    row_number=line_number,
+                    field=name,
+                )
+            )
+            continue
+        if name not in row and len(row) >= MAX_LDIF_ATTRIBUTES:
+            diagnostics.append(
+                _diagnostic(
+                    "error",
+                    "ldif_too_many_attributes",
+                    f"LDIF entry exceeds the {MAX_LDIF_ATTRIBUTES}-attribute limit.",
+                    row_number=line_number,
+                )
+            )
+            continue
+        value, value_diagnostic = _decode_value(raw_value, attribute=name, line_number=line_number)
+        if value_diagnostic is not None:
+            diagnostics.append(value_diagnostic)
+        if value is None:
+            continue
+        values = row.setdefault(name, [])
+        if len(values) >= MAX_LDIF_VALUES_PER_ATTRIBUTE:
+            diagnostics.append(
+                _diagnostic(
+                    "error",
+                    "ldif_too_many_values",
+                    f"LDIF attribute {name!r} exceeds the value limit.",
+                    row_number=line_number,
+                    field=name,
+                )
+            )
+            continue
+        values.append(value)
+    return row, diagnostics
+
+
+def _decode_value(
+    raw_value: bytes,
+    *,
+    attribute: str,
+    line_number: int,
+) -> tuple[str | None, dict[str, Any] | None]:
+    if raw_value.startswith(b":"):
+        encoded = raw_value[1:].lstrip(b" ")
+        try:
+            decoded = base64.b64decode(encoded, validate=True)
+        except (binascii.Error, ValueError):
+            return None, _diagnostic(
+                "error",
+                "ldif_invalid_base64",
+                f"LDIF attribute {attribute!r} contains invalid base64.",
+                row_number=line_number,
+                field=attribute,
+            )
+        try:
+            return decoded.decode("utf-8"), None
+        except UnicodeDecodeError:
+            return None, _diagnostic(
+                "warning",
+                "ldif_binary_value_ignored",
+                f"Binary LDIF attribute {attribute!r} was ignored; binary data is never projected into contacts.",
+                row_number=line_number,
+                field=attribute,
+            )
+    if raw_value.startswith(b"<"):
+        return None, _diagnostic(
+            "warning",
+            "ldif_url_value_ignored",
+            f"External LDIF URL value for {attribute!r} was ignored; imports never fetch referenced content.",
+            row_number=line_number,
+            field=attribute,
+        )
+    value_bytes = raw_value[1:] if raw_value.startswith(b" ") else raw_value
+    try:
+        return value_bytes.decode("utf-8"), None
+    except UnicodeDecodeError:
+        return None, _diagnostic(
+            "error",
+            "ldif_invalid_utf8",
+            f"LDIF attribute {attribute!r} is not valid UTF-8.",
+            row_number=line_number,
+            field=attribute,
+        )
+
+
+def _canonical_record(row: dict[str, Any]) -> bytes:
+    lines = []
+    for name in sorted(key for key in row if not key.startswith("__")):
+        values = row[name] if isinstance(row[name], list) else [row[name]]
+        lines.extend(f"{name}:{value}" for value in values)
+    return "\n".join(lines).encode("utf-8")
+
+
+def _first(value: object) -> str:
+    if isinstance(value, list):
+        return str(value[0]) if value else ""
+    return str(value or "")
+
+
+def _diagnostic(
+    severity: str,
+    code: str,
+    message: str,
+    *,
+    row_number: int | None = None,
+    field: str | None = None,
+) -> dict[str, Any]:
+    return {
+        "severity": severity,
+        "code": code,
+        "message": message,
+        "row_number": row_number,
+        "field": field,
+        "details": {},
+    }
+
+
+__all__ = ["LdifChangeRecordPolicy", "parse_ldif_rows"]

src/govoplan_addresses/backend/manifest.py

@@ -440,17 +440,19 @@ manifest = ModuleManifest(
         ),
         DocumentationTopic(
             id="addresses.tabular-imports",
-            title="CSV and XLSX contact imports",
-            summary="Preview and apply reusable, versioned contact mappings without silent row loss.",
+            title="CSV, XLSX, and LDIF contact imports",
+            summary="Preview and apply reusable, versioned contact mappings without silent row or entry loss.",
             body=(
-                "CSV and XLSX files can be mapped with scoped, reusable profile versions. Each preview validates headers, "
-                "encodings, source keys, duplicates, blank values, workbook limits, and contact identity before any mutation. "
+                "CSV, XLSX, and LDIF files can be mapped with scoped, reusable profile versions. Each preview validates headers or attributes, "
+                "encodings, source keys, duplicates, blank values, format limits, and contact identity before any mutation. "
                 "The reviewed input hash and plan hash are retained with row-level effects and diagnostics. Apply is idempotent, "
                 "rejects contacts changed after preview, and records sufficient evidence for a guarded rollback. A persisted run "
                 "can be reopened with its run link after navigation or reload; previewed, applied, rolled-back, expired, and "
                 "unavailable states remain explicit. Both apply and rollback submit the reviewed plan hash. Missing, expired, "
                 "hidden, and cross-tenant runs disclose no source payload. XLSX formulas, macros, and legacy workbook formats "
-                "are never executed or imported."
+                "are never executed or imported. LDIF folded lines, UTF-8 and base64 text, repeated attributes, and comments are parsed; "
+                "binary and URL values are never projected or fetched. Change records default to rejected diagnostics and may only be ignored "
+                "or treat add records as static entries through an explicit profile policy."
             ),
             layer="configured",
             documentation_types=("admin", "user"),