Add governed tabular and LDAP address sources

This commit is contained in:
2026-08-02 15:39:34 +02:00
parent e60339a5bf
commit 2c421022d4
19 changed files with 3846 additions and 78 deletions
+518 -43
View File
@@ -38,6 +38,14 @@ from govoplan_addresses.backend.carddav import (
AddressCardDAVSyncUnsupported,
ensure_collection_url,
)
from govoplan_addresses.backend.ldap import (
AddressLdapClient,
AddressLdapError,
)
from govoplan_addresses.backend.ldap_schemas import (
AddressLdapConnectionRequest,
AddressLdapSourceCreateRequest,
)
from govoplan_addresses.backend.db.models import (
AddressBook,
AddressList,
@@ -142,6 +150,8 @@ class AddressSyncPlanItem:
raw_vcard: str | None = None
parsed_payload: ContactCreateRequest | None = None
source_revision: str | None = None
raw_payload: str | None = None
source_details: dict[str, Any] = field(default_factory=dict)
@dataclass(slots=True)
@@ -386,6 +396,8 @@ def create_sync_source(
raise AddressBookError("Sync source display name is required.")
if connector_type.casefold() == "carddav" and not trusted_connector_metadata:
_assert_api_carddav_metadata_safe(payload.metadata)
if connector_type.casefold() in {"ldap", "active_directory"} and not trusted_connector_metadata:
_assert_api_ldap_metadata_safe(payload.metadata)
read_only = _read_only_from_sync_direction(payload.sync_direction, payload.read_only)
sync_source = AddressSyncSource(
tenant_id=book.tenant_id,
@@ -449,6 +461,9 @@ def update_sync_source(
if sync_source.connector_type.casefold() == "carddav":
_assert_api_carddav_metadata_safe(metadata)
metadata = _merge_server_owned_carddav_metadata(sync_source.metadata_, metadata)
elif sync_source.connector_type.casefold() in {"ldap", "active_directory"}:
_assert_api_ldap_metadata_safe(metadata)
metadata = _merge_server_owned_ldap_metadata(sync_source.metadata_, metadata)
sync_source.metadata_ = metadata
sync_source.updated_by_account_id = _account_id(principal)
_apply_sync_source_to_book(sync_source.address_book, sync_source)
@@ -774,7 +789,7 @@ def _apply_payload_sync_conflict(session: Session, principal: ApiPrincipal, conf
parsed_payload=payload,
source_revision=str(metadata.get("source_revision") or remote_value.get("etag") or "") or None,
)
_upsert_carddav_contact(session, principal, conflict.sync_source, item)
_upsert_remote_contact(session, principal, conflict.sync_source, item)
def get_visible_sync_conflict(session: Session, principal: ApiPrincipal, conflict_id: str) -> AddressSyncConflict:
@@ -849,6 +864,80 @@ def create_carddav_sync_source(
return source
def discover_ldap_base_dns(
session: Session,
principal: ApiPrincipal,
payload: AddressLdapConnectionRequest,
*,
client: AddressLdapClient | None = None,
) -> tuple[str, ...]:
_assert_reusable_credential_ref(payload.credential_ref)
ldap_client = client or _ldap_client_from_connection_payload(
session,
principal,
payload,
)
return ldap_client.discover_base_dns()
def create_ldap_sync_source(
session: Session,
principal: ApiPrincipal,
address_book_id: str,
payload: AddressLdapSourceCreateRequest,
) -> AddressSyncSource:
_assert_reusable_credential_ref(payload.credential_ref)
# Constructor validation rejects plaintext LDAP and embedded URL credentials
# without opening a network connection.
AddressLdapClient(
url=payload.url,
bind_dn=payload.bind_dn,
start_tls=payload.start_tls,
connect_timeout=payload.connect_timeout,
receive_timeout=payload.receive_timeout,
)
metadata = {
"ldap": {
"url": payload.url.strip(),
"base_dn": payload.base_dn.strip(),
"search_filter": payload.search_filter.strip(),
"start_tls": payload.start_tls,
"connect_timeout": payload.connect_timeout,
"receive_timeout": payload.receive_timeout,
"page_size": payload.page_size,
"max_entries": payload.max_entries,
"attribute_map": dict(payload.attribute_map),
"bind_dn": _trim(payload.bind_dn),
"credential_ref": _trim(payload.credential_ref),
}
}
source = create_sync_source(
session,
principal,
address_book_id,
AddressSyncSourceCreateRequest(
connector_type="ldap",
display_name=payload.display_name,
external_account_ref=payload.url,
external_address_book_ref=payload.base_dn,
sync_direction="read_only",
read_only=True,
metadata=metadata,
),
trusted_connector_metadata=True,
)
reusable = _resolve_core_address_credential(
session,
tenant_id=principal.tenant_id,
source_id=source.id,
credential_ref=payload.credential_ref,
)
if reusable is not None and not metadata["ldap"].get("bind_dn"):
metadata["ldap"]["bind_dn"] = _credential_username(reusable)
source.metadata_ = metadata
return source
def preview_sync_source(
session: Session,
principal: ApiPrincipal,
@@ -857,20 +946,27 @@ def preview_sync_source(
force_full: bool = False,
password: str | None = None,
bearer_token: str | None = None,
client: AddressCardDAVClient | None = None,
client: AddressCardDAVClient | AddressLdapClient | None = None,
) -> AddressSyncPlan:
sync_source = get_visible_sync_source(session, principal, sync_source_id)
if sync_source.connector_type != "carddav":
raise AddressBookError(f"Preview is not implemented for {sync_source.connector_type} sync sources.")
return _build_carddav_sync_plan(
session,
principal,
sync_source,
force_full=force_full,
password=password,
bearer_token=bearer_token,
client=client,
)
if sync_source.connector_type == "carddav":
return _build_carddav_sync_plan(
session,
principal,
sync_source,
force_full=force_full,
password=password,
bearer_token=bearer_token,
client=client, # type: ignore[arg-type]
)
if sync_source.connector_type in {"ldap", "active_directory"}:
return _build_ldap_sync_plan(
session,
principal,
sync_source,
client=client, # type: ignore[arg-type]
)
raise AddressBookError(f"Preview is not implemented for {sync_source.connector_type} sync sources.")
def run_sync_source(
@@ -881,23 +977,38 @@ def run_sync_source(
force_full: bool = False,
password: str | None = None,
bearer_token: str | None = None,
client: AddressCardDAVClient | None = None,
client: AddressCardDAVClient | AddressLdapClient | None = None,
) -> AddressSyncPlan:
sync_source = start_sync_attempt(session, principal, sync_source_id)
write_client = client
if sync_source.connector_type == "carddav" and write_client is None:
write_client = _carddav_client_for_source(session, sync_source, password=password, bearer_token=bearer_token)
try:
plan = _build_carddav_sync_plan(
if sync_source.connector_type == "carddav":
plan = _build_carddav_sync_plan(
session,
principal,
sync_source,
force_full=force_full,
password=password,
bearer_token=bearer_token,
client=write_client, # type: ignore[arg-type]
)
elif sync_source.connector_type in {"ldap", "active_directory"}:
plan = _build_ldap_sync_plan(
session,
principal,
sync_source,
client=write_client, # type: ignore[arg-type]
)
else:
raise AddressBookError(f"Sync is not implemented for {sync_source.connector_type} sources.")
_apply_address_sync_plan(
session,
principal,
sync_source,
force_full=force_full,
password=password,
bearer_token=bearer_token,
client=write_client,
plan,
client=write_client if sync_source.connector_type == "carddav" else None, # type: ignore[arg-type]
)
_apply_address_sync_plan(session, principal, plan, client=write_client)
status = "conflict" if plan.stats.conflicts else "succeeded"
if plan.stats.errors:
status = "failed"
@@ -925,6 +1036,275 @@ def run_sync_source(
raise
def _build_ldap_sync_plan(
session: Session,
principal: ApiPrincipal,
sync_source: AddressSyncSource,
*,
client: AddressLdapClient | None,
) -> AddressSyncPlan:
settings = _ldap_metadata(sync_source.metadata_)
if not settings:
raise AddressBookError("LDAP sync source configuration is missing.")
ldap_client = client or _ldap_client_for_source(session, sync_source)
attribute_map = {
str(key): str(value)
for key, value in dict(settings.get("attribute_map") or {}).items()
if str(key).strip() and str(value).strip()
}
source_key_attribute = attribute_map.get("source_key")
if not source_key_attribute:
raise AddressBookError("LDAP source mapping requires a stable source_key attribute.")
attributes = tuple(
dict.fromkeys(
[
*attribute_map.values(),
"entryUUID",
"objectGUID",
"modifyTimestamp",
"uSNChanged",
"entryCSN",
]
)
)
try:
result = ldap_client.search(
base_dn=str(settings.get("base_dn") or sync_source.external_address_book_ref or ""),
search_filter=str(settings.get("search_filter") or "(objectClass=person)"),
attributes=attributes,
page_size=int(settings.get("page_size") or 500),
max_entries=int(settings.get("max_entries") or 10_000),
)
except AddressLdapError as exc:
raise AddressBookError(str(exc)) from exc
stats = AddressSyncPlanStats(full_sync=True)
plan = AddressSyncPlan(sync_source=sync_source, stats=stats)
existing = {
str(contact.source_ref): contact
for contact in _ldap_contacts_for_source(session, sync_source)
if contact.source_ref
}
observed_refs: set[str] = set()
revision_rows: list[dict[str, str]] = []
for entry in result.entries:
serialized_attributes = _json_safe_ldap_attributes(entry.attributes)
source_key = _ldap_scalar(entry.attributes.get(source_key_attribute))
if not source_key:
source_key = entry.dn.strip()
if not source_key:
_add_sync_plan_item(
plan,
AddressSyncPlanItem(
action="error",
message="LDAP entry has neither the configured source key nor a DN.",
),
)
continue
source_ref = f"ldap:{sync_source.id}:{hashlib.sha256(source_key.encode()).hexdigest()}"
if source_ref in observed_refs:
_add_sync_plan_item(
plan,
AddressSyncPlanItem(
action="error",
href=source_ref,
remote_uid=source_key,
message="LDAP search returned a duplicate stable source key.",
),
)
continue
observed_refs.add(source_ref)
source_revision = _ldap_source_revision(
entry.attributes,
attribute_map=attribute_map,
serialized_attributes=serialized_attributes,
)
revision_rows.append({"source_key": source_key, "revision": source_revision})
try:
payload = _ldap_contact_payload(
entry.attributes,
attribute_map=attribute_map,
sync_source=sync_source,
source_key=source_key,
source_revision=source_revision,
dn=entry.dn,
)
except (AddressBookError, ValueError) as exc:
_add_sync_plan_item(
plan,
AddressSyncPlanItem(
action="error",
href=source_ref,
remote_uid=source_key,
message=str(exc),
source_revision=source_revision,
),
)
continue
local = existing.get(source_ref)
action = "create"
if local is not None:
comparable = payload.model_dump(mode="json", exclude={"provenance"})
action = (
"unchanged"
if local.deleted_at is None
and _contact_payload_for_conflict(local) == comparable
and local.source_revision == source_revision
else "update"
)
_add_sync_plan_item(
plan,
AddressSyncPlanItem(
action=action,
href=source_ref,
remote_uid=source_key,
contact_id=local.id if local is not None else None,
display_name=payload.display_name,
parsed_payload=payload,
source_revision=source_revision,
raw_payload=json.dumps(serialized_attributes, sort_keys=True, ensure_ascii=True),
source_details={"dn": entry.dn, "source_key": source_key},
),
)
if result.complete:
for source_ref, contact in existing.items():
if source_ref not in observed_refs and contact.deleted_at is None:
_add_sync_plan_item(
plan,
AddressSyncPlanItem(
action="delete",
href=source_ref,
remote_uid=str((contact.provenance or {}).get("ldap", {}).get("source_key") or "") or None,
contact_id=contact.id,
display_name=contact.display_name,
message="LDAP authoritative source no longer contains this contact.",
),
)
else:
_add_sync_plan_item(
plan,
AddressSyncPlanItem(
action="error",
message="LDAP result reached the configured entry limit; absence-based deletes are suppressed.",
),
)
stats.remote_revision = hashlib.sha256(
json.dumps(sorted(revision_rows, key=lambda item: item["source_key"]), sort_keys=True).encode()
).hexdigest()
return plan
def _ldap_contact_payload(
attributes: dict[str, Any],
*,
attribute_map: dict[str, str],
sync_source: AddressSyncSource,
source_key: str,
source_revision: str,
dn: str,
) -> ContactCreateRequest:
def scalar(target: str) -> str | None:
attribute = attribute_map.get(target)
return _ldap_scalar(attributes.get(attribute)) if attribute else None
given_name = scalar("given_name")
family_name = scalar("family_name")
email = scalar("email")
organization = scalar("organization")
display_name = scalar("display_name") or " ".join(
value for value in (given_name, family_name) if value
) or email or organization
if not display_name:
raise AddressBookError(f'LDAP entry "{dn or source_key}" has no mapped contact identity.')
phone = scalar("phone")
postal = {
target: scalar(target)
for target in ("street", "postal_code", "locality", "region", "country")
}
tag_attribute = attribute_map.get("tags")
tags = [str(item).strip() for item in _ldap_values(attributes.get(tag_attribute)) if str(item).strip()] if tag_attribute else []
return ContactCreateRequest(
display_name=display_name,
given_name=given_name,
family_name=family_name,
organization=organization,
role_title=scalar("role_title"),
note=scalar("note"),
tags=tags,
emails=[ContactEmailPayload(email=email, is_primary=True)] if email else [],
phones=[ContactPhonePayload(phone=phone, is_primary=True)] if phone else [],
postal_addresses=[ContactPostalAddressPayload(**postal, is_primary=True)] if any(postal.values()) else [],
provenance={
"ldap": {
"sync_source_id": sync_source.id,
"dn": dn,
"source_key": source_key,
"source_revision": source_revision,
"authority": "external_authoritative",
}
},
)
def _ldap_source_revision(
attributes: dict[str, Any],
*,
attribute_map: dict[str, str],
serialized_attributes: dict[str, Any],
) -> str:
configured = attribute_map.get("source_revision")
for attribute in (configured, "modifyTimestamp", "uSNChanged", "entryCSN"):
if attribute:
value = _ldap_scalar(attributes.get(attribute))
if value:
return value
return hashlib.sha256(
json.dumps(serialized_attributes, sort_keys=True, ensure_ascii=True).encode()
).hexdigest()
def _ldap_values(value: Any) -> tuple[Any, ...]:
if value is None:
return ()
if isinstance(value, (list, tuple, set)):
return tuple(value)
return (value,)
def _ldap_scalar(value: Any) -> str | None:
values = _ldap_values(value)
if not values:
return None
selected = values[0]
if isinstance(selected, bytes):
return selected.hex()
normalized = str(selected).strip()
return normalized or None
def _json_safe_ldap_attributes(attributes: dict[str, Any]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in attributes.items():
values = _ldap_values(value)
normalized = [item.hex() if isinstance(item, bytes) else str(item) for item in values]
result[str(key)] = normalized if isinstance(value, (list, tuple, set)) else (normalized[0] if normalized else None)
return result
def _ldap_contacts_for_source(session: Session, sync_source: AddressSyncSource) -> list[Contact]:
return (
session.query(Contact)
.filter(
Contact.address_book_id == sync_source.address_book_id,
Contact.source_kind.in_(("ldap", "active_directory")),
Contact.source_ref.like(f"ldap:{sync_source.id}:%"),
)
.order_by(Contact.id.asc())
.all()
)
def _build_carddav_sync_plan(
session: Session,
principal: ApiPrincipal,
@@ -1437,10 +1817,10 @@ def _add_sync_plan_item(plan: AddressSyncPlan, item: AddressSyncPlanItem) -> Non
def _apply_address_sync_plan(session: Session, principal: ApiPrincipal, plan: AddressSyncPlan, *, client: AddressCardDAVClient | None = None) -> None:
for item in plan.items:
if item.action == "create":
contact = _upsert_carddav_contact(session, principal, plan.sync_source, item)
contact = _upsert_remote_contact(session, principal, plan.sync_source, item)
item.contact_id = contact.id
elif item.action == "update":
contact = _upsert_carddav_contact(session, principal, plan.sync_source, item)
contact = _upsert_remote_contact(session, principal, plan.sync_source, item)
item.contact_id = contact.id
elif item.action == "delete" and item.contact_id:
contact = get_visible_contact(session, principal, item.contact_id, include_deleted=True)
@@ -1612,14 +1992,14 @@ def _decrement_sync_plan_stats(plan: AddressSyncPlan, action: str) -> None:
plan.stats.deleted = max(0, plan.stats.deleted - 1)
def _upsert_carddav_contact(
def _upsert_remote_contact(
session: Session,
principal: ApiPrincipal,
sync_source: AddressSyncSource,
item: AddressSyncPlanItem,
) -> Contact:
if item.parsed_payload is None:
raise AddressBookError("Remote vCard payload is missing.")
raise AddressBookError("Mapped remote contact payload is missing.")
contact = None
if item.contact_id:
contact = get_visible_contact(session, principal, item.contact_id, include_deleted=True)
@@ -1645,18 +2025,30 @@ def _upsert_carddav_contact(
contact.tags = _normalize_tags(item.parsed_payload.tags)
contact.source_kind = sync_source.connector_type
contact.source_ref = item.href
contact.source_payload_kind = "vcard"
contact.source_payload_raw = item.raw_vcard
contact.source_payload_kind = "vcard" if sync_source.connector_type == "carddav" else "ldap-entry"
contact.source_payload_raw = item.raw_vcard if sync_source.connector_type == "carddav" else None
contact.source_revision = item.source_revision
contact.provenance = {
**(item.parsed_payload.provenance or {}),
"carddav": {
"sync_source_id": sync_source.id,
"href": item.href,
"remote_uid": item.remote_uid,
"etag": item.etag,
},
}
if sync_source.connector_type == "carddav":
source_provenance = {
"carddav": {
"sync_source_id": sync_source.id,
"href": item.href,
"remote_uid": item.remote_uid,
"etag": item.etag,
}
}
else:
source_provenance = {
"ldap": {
"sync_source_id": sync_source.id,
"source_ref": item.href,
"source_key": item.remote_uid,
"source_revision": item.source_revision,
"dn": item.source_details.get("dn"),
"authority": "external_authoritative",
}
}
contact.provenance = {**(item.parsed_payload.provenance or {}), **source_provenance}
contact.deleted_at = None
contact.updated_by_account_id = _account_id(principal)
_replace_emails(contact, item.parsed_payload.emails)
@@ -1727,6 +2119,52 @@ def _local_contact_changed_after_last_sync(contact: Contact, sync_source: Addres
return bool(contact.updated_at and contact.updated_at > sync_source.last_success_at)
def _ldap_client_from_connection_payload(
session: Session,
principal: ApiPrincipal,
payload: AddressLdapConnectionRequest,
) -> AddressLdapClient:
reusable = _resolve_core_address_credential(
session,
tenant_id=principal.tenant_id,
source_id=None,
credential_ref=payload.credential_ref,
)
bind_dn = _trim(payload.bind_dn) or _credential_username(reusable)
password = _credential_secret(reusable, auth_type="basic") if reusable is not None else None
return AddressLdapClient(
url=payload.url,
bind_dn=bind_dn,
password=password,
start_tls=payload.start_tls,
connect_timeout=payload.connect_timeout,
receive_timeout=payload.receive_timeout,
)
def _ldap_client_for_source(
session: Session,
sync_source: AddressSyncSource,
) -> AddressLdapClient:
settings = _ldap_metadata(sync_source.metadata_)
reusable = _resolve_core_address_credential(
session,
tenant_id=sync_source.tenant_id or "",
source_id=sync_source.id,
credential_ref=settings.get("credential_ref"),
)
bind_dn = _trim(str(settings.get("bind_dn") or "")) or _credential_username(reusable)
password = _credential_secret(reusable, auth_type="basic") if reusable is not None else None
return AddressLdapClient(
url=str(settings.get("url") or sync_source.external_account_ref or ""),
bind_dn=bind_dn,
password=password,
start_tls=bool(settings.get("start_tls", True)),
connect_timeout=int(settings.get("connect_timeout") or 10),
receive_timeout=int(settings.get("receive_timeout") or 30),
)
def _carddav_client_from_payload(
session: Session,
principal: ApiPrincipal,
@@ -1934,13 +2372,17 @@ def resolve_trusted_deployment_carddav_credential_ref(credential_ref: str) -> st
def public_address_sync_metadata(metadata: object) -> dict[str, Any]:
payload = copy.deepcopy(metadata) if isinstance(metadata, dict) else {}
auth = payload.get("carddav")
if not isinstance(auth, dict):
return payload
had_credential = bool(auth.get("secret_encrypted") or auth.get("credential_ref"))
auth["credential_envelope_id"] = _core_credential_id(auth.get("credential_ref"))
auth.pop("secret_encrypted", None)
auth.pop("credential_ref", None)
auth["has_credential"] = had_credential
if isinstance(auth, dict):
had_credential = bool(auth.get("secret_encrypted") or auth.get("credential_ref"))
auth["credential_envelope_id"] = _core_credential_id(auth.get("credential_ref"))
auth.pop("secret_encrypted", None)
auth.pop("credential_ref", None)
auth["has_credential"] = had_credential
ldap = payload.get("ldap")
if isinstance(ldap, dict):
credential_ref = ldap.pop("credential_ref", None)
ldap["credential_envelope_id"] = _core_credential_id(credential_ref)
ldap["has_credential"] = bool(credential_ref)
return payload
@@ -1952,6 +2394,39 @@ def _assert_no_caller_carddav_credential_ref(credential_ref: str | None) -> None
)
def _assert_reusable_credential_ref(credential_ref: str | None) -> None:
_assert_no_caller_carddav_credential_ref(credential_ref)
def _ldap_metadata(metadata: object) -> dict[str, Any]:
if not isinstance(metadata, dict):
return {}
ldap = metadata.get("ldap")
return ldap if isinstance(ldap, dict) else {}
def _assert_api_ldap_metadata_safe(metadata: object) -> None:
ldap = _ldap_metadata(metadata)
forbidden = {"credential_ref", "password", "secret", "bind_password"}.intersection(ldap)
if forbidden:
raise AddressBookError(
"LDAP credential references and secrets are server-managed; use the LDAP source endpoint."
)
def _merge_server_owned_ldap_metadata(existing: object, incoming: dict[str, Any]) -> dict[str, Any]:
merged = copy.deepcopy(incoming)
existing_ldap = _ldap_metadata(existing)
credential_ref = existing_ldap.get("credential_ref")
if credential_ref:
ldap = merged.get("ldap")
if not isinstance(ldap, dict):
ldap = {}
merged["ldap"] = ldap
ldap["credential_ref"] = credential_ref
return merged
def _carddav_auth_metadata(metadata: object) -> dict[str, Any]:
if not isinstance(metadata, dict):
return {}