GovOPlaN/govoplan-addresses
2
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add governed LDIF contact imports

Browse Source
This commit is contained in:
zemion
2026-08-20 11:19:23 +02:00
parent ec32ad2c37
commit 147f34c5c1
9 changed files with 732 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_addresses/backend/imports.py
+153 -20
View File
@@ -24,6 +24,7 @@ from govoplan_addresses.backend.import_schemas import (
AddressImportProfileUpdateRequest,
AddressImportRollbackRequest,
)
from govoplan_addresses.backend.ldif import parse_ldif_rows
from govoplan_addresses.backend.schemas import (
ContactCreateRequest,
ContactEmailPayload,
@@ -199,6 +200,7 @@ def preview_address_import(
input_hash=input_hash,
rows=rows,
config=config,
source_format=profile.source_format,
)
diagnostics = [*parse_diagnostics, *map_diagnostics]
statistics = dict(Counter(item["action"] for item in plan_data))
@@ -501,13 +503,24 @@ def _parse_rows(
filename: str,
source_format: str,
config: AddressImportConfiguration,
) -> tuple[list[tuple[int, dict[str, str]]], list[dict[str, Any]]]:
) -> tuple[list[tuple[int, dict[str, Any]]], list[dict[str, Any]]]:
if source_format == "csv":
return _parse_csv(raw, config=config)
if source_format == "xlsx":
if not filename.casefold().endswith(".xlsx"):
raise AddressBookError("XLSX imports require an .xlsx file; macros and legacy workbooks are not accepted.")
return _parse_xlsx(raw, config=config)
if source_format == "ldif":
if not filename.casefold().endswith((".ldif", ".ldi")):
raise AddressBookError("LDIF imports require an .ldif or .ldi file.")
try:
return parse_ldif_rows(
raw,
max_entries=config.max_rows,
change_record_policy=config.ldif_change_record_policy,
)
except ValueError as exc:
raise AddressBookError(str(exc)) from exc
raise AddressBookError(f"Unsupported address import format: {source_format!r}.")
@@ -600,32 +613,81 @@ def _row_dict(headers: list[str], values: list[Any]) -> dict[str, str]:
}
def _column_key(value: object, *, casefold: bool) -> str:
column = str(value).strip()
return column.casefold() if casefold else column
def _row_value(row: dict[str, Any], column: str, *, casefold: bool) -> object:
if column in row:
return row[column]
if not casefold:
return ""
expected = column.casefold()
for key, value in row.items():
if key.casefold() == expected:
return value
return ""
def _import_values(value: object) -> list[str]:
raw_values = value if isinstance(value, list) else [value]
return [str(item).strip() for item in raw_values if item is not None and str(item).strip()]
def _first_import_value(value: object) -> str:
values = _import_values(value)
return values[0] if values else ""
def _plan_rows(
session: Session,
*,
book_id: str,
profile: AddressImportProfile,
input_hash: str,
rows: list[tuple[int, dict[str, str]]],
rows: list[tuple[int, dict[str, Any]]],
config: AddressImportConfiguration,
source_format: str,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
diagnostics: list[dict[str, Any]] = []
plan: list[dict[str, Any]] = []
headers = set(rows[0][1]) if rows else set()
referenced_columns = set(config.field_mappings.values())
casefold_columns = source_format == "ldif"
headers = {
_column_key(column, casefold=casefold_columns)
for column in (rows[0][1] if rows else set())
if not str(column).startswith("__")
}
referenced_columns = {
_column_key(column, casefold=casefold_columns)
for column in config.field_mappings.values()
}
key_column = config.source_key_column or config.field_mappings["source_key"]
normalized_key_column = _column_key(key_column, casefold=casefold_columns)
if config.source_key_column:
referenced_columns.add(config.source_key_column)
referenced_columns.add(_column_key(config.source_key_column, casefold=casefold_columns))
missing_columns = sorted(referenced_columns.difference(headers))
for column in missing_columns:
diagnostics.append(_diagnostic("error", "missing_column", f'Configured column "{column}" is missing.', field=column))
if missing_columns:
optional_ldif_attribute = casefold_columns and column != normalized_key_column
diagnostics.append(
_diagnostic(
"warning" if optional_ldif_attribute else "error",
"missing_attribute" if optional_ldif_attribute else "missing_column",
(
f'Configured LDIF attribute "{column}" is absent from this file.'
if optional_ldif_attribute
else f'Configured column "{column}" is missing.'
),
field=column,
)
)
if normalized_key_column in missing_columns or (missing_columns and not casefold_columns):
return [], diagnostics
key_column = config.source_key_column or config.field_mappings["source_key"]
keyed_rows: list[tuple[int, dict[str, str], str]] = []
keyed_rows: list[tuple[int, dict[str, Any], str]] = []
key_counts: Counter[str] = Counter()
for row_number, row in rows:
key = row.get(key_column, "").strip()
key = _first_import_value(_row_value(row, key_column, casefold=casefold_columns)).strip()
if not key:
diagnostics.append(_diagnostic("error", "missing_source_key", "Stable source key is blank.", row_number=row_number, field=key_column))
plan.append(_plan_effect(row_number, "conflict", source_key=None, message="Stable source key is blank."))
@@ -651,11 +713,23 @@ def _plan_rows(
plan.append(_plan_effect(row_number, "ignored", source_key=key, message="Duplicate row ignored by profile policy."))
continue
mapped, row_diagnostics = _mapped_fields(row_number, row, config=config)
mapped, row_diagnostics = _mapped_fields(
row_number,
row,
config=config,
casefold_columns=casefold_columns,
)
diagnostics.extend(row_diagnostics)
source_ref = f"import:{profile.profile_key}:{key}"
existing = _contact_by_source_ref(session, book_id, source_ref)
payload = _payload_from_mapped(mapped, profile=profile, input_hash=input_hash, row_number=row_number, source_key=key)
payload = _payload_from_mapped(
mapped,
profile=profile,
input_hash=input_hash,
row_number=row_number,
source_key=key,
source_row=row,
)
display_name = payload.get("display_name") or payload.get("email") or key
if any(item["severity"] == "error" for item in row_diagnostics):
plan.append(_plan_effect(row_number, "conflict", source_key=key, display_name=display_name, source_ref=source_ref, payload=payload, message="Row validation failed."))
@@ -689,23 +763,46 @@ def _plan_rows(
def _mapped_fields(
row_number: int,
row: dict[str, str],
row: dict[str, Any],
*,
config: AddressImportConfiguration,
casefold_columns: bool = False,
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
mapped: dict[str, Any] = {}
diagnostics: list[dict[str, Any]] = []
for target, column in config.field_mappings.items():
if target == "source_key":
continue
value = row.get(column, "").strip()
if not value:
values = _import_values(_row_value(row, column, casefold=casefold_columns))
if not values:
if config.blank_value_policy == "reject":
diagnostics.append(_diagnostic("error", "blank_value", f'Column "{column}" is blank.', row_number=row_number, field=target))
elif config.blank_value_policy == "clear":
mapped[target] = [] if target == "tags" else None
continue
mapped[target] = [item.strip() for item in value.split(",") if item.strip()] if target == "tags" else value
if target == "tags":
mapped[target] = list(
dict.fromkeys(
item.strip()
for value in values
for item in value.split(",")
if item.strip()
)
)
elif target in {"email", "phone"}:
mapped[target] = values if len(values) > 1 else values[0]
else:
mapped[target] = values[0]
if len(values) > 1:
diagnostics.append(
_diagnostic(
"warning",
"multiple_values_truncated",
f'Attribute "{column}" has multiple values; only the first maps to {target!r}.',
row_number=row_number,
field=target,
)
)
if config.default_tags:
mapped["tags"] = list(dict.fromkeys([*(mapped.get("tags") or []), *config.default_tags]))
if not any(mapped.get(name) for name in ("display_name", "given_name", "family_name", "email", "organization")):
@@ -720,8 +817,22 @@ def _payload_from_mapped(
input_hash: str,
row_number: int,
source_key: str,
source_row: dict[str, Any] | None = None,
) -> dict[str, Any]:
display_name = mapped.get("display_name") or " ".join(filter(None, [mapped.get("given_name"), mapped.get("family_name")])) or mapped.get("email") or mapped.get("organization")
display_name = (
_first_import_value(mapped.get("display_name"))
or " ".join(
filter(
None,
[
_first_import_value(mapped.get("given_name")),
_first_import_value(mapped.get("family_name")),
],
)
)
or _first_import_value(mapped.get("email"))
or _first_import_value(mapped.get("organization"))
)
payload: dict[str, Any] = {
key: mapped.get(key)
for key in ("given_name", "family_name", "organization", "role_title", "note", "tags")
@@ -729,9 +840,17 @@ def _payload_from_mapped(
}
payload["display_name"] = display_name
if "email" in mapped:
payload["emails"] = [] if mapped["email"] is None else [ContactEmailPayload(email=mapped["email"], is_primary=True).model_dump(mode="json")]
email_values = _import_values(mapped["email"])
payload["emails"] = [
ContactEmailPayload(email=value, is_primary=index == 0).model_dump(mode="json")
for index, value in enumerate(email_values)
]
if "phone" in mapped:
payload["phones"] = [] if mapped["phone"] is None else [ContactPhonePayload(phone=mapped["phone"], is_primary=True).model_dump(mode="json")]
phone_values = _import_values(mapped["phone"])
payload["phones"] = [
ContactPhonePayload(phone=value, is_primary=index == 0).model_dump(mode="json")
for index, value in enumerate(phone_values)
]
postal_keys = {"street", "postal_code", "locality", "region", "country"}
if postal_keys.intersection(mapped):
postal = {key: mapped.get(key) for key in postal_keys if key in mapped}
@@ -746,6 +865,8 @@ def _payload_from_mapped(
"source_key": source_key,
"locale": profile.configuration.get("locale"),
"visibility": mapped.get("visibility"),
"source_format": profile.source_format,
"source_record_hash": (source_row or {}).get("__ldif_record_hash"),
}
}
return ContactCreateRequest.model_validate(payload).model_dump(
@@ -770,7 +891,19 @@ def _changed_fields(contact: Contact, mapped: dict[str, Any]) -> list[str]:
if contact.postal_addresses:
postal = contact.postal_addresses[0]
current.update({key: getattr(postal, key) for key in ("street", "postal_code", "locality", "region", "country")})
return sorted(key for key, value in mapped.items() if key != "visibility" and current.get(key) != value)
changed: list[str] = []
for key, value in mapped.items():
if key == "visibility":
continue
if key == "email" and isinstance(value, list):
current_value: Any = [item.email for item in contact.emails]
elif key == "phone" and isinstance(value, list):
current_value = [item.phone for item in contact.phones]
else:
current_value = current.get(key)
if current_value != value:
changed.append(key)
return sorted(changed)
def _plan_effect(