Add governed tabular and LDAP address sources

2026-08-02 15:39:34 +02:00
parent e60339a5bf
commit 2c421022d4
19 changed files with 3846 additions and 78 deletions
@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import unittest
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from govoplan_addresses.backend.db.models import AddressBook, AddressSyncSource, Contact
+from govoplan_addresses.backend.ldap import (
+    AddressLdapClient,
+    AddressLdapEntry,
+    AddressLdapError,
+    AddressLdapSearchResult,
+)
+from govoplan_addresses.backend.ldap_schemas import AddressLdapSourceCreateRequest
+from govoplan_addresses.backend.service import (
+    create_ldap_sync_source,
+    preview_sync_source,
+    run_sync_source,
+)
+from govoplan_core.db.base import Base
+
+
+class Principal:
+    account_id = "account-1"
+    group_ids = frozenset()
+
+    @property
+    def tenant_id(self) -> str:
+        return "tenant-1"
+
+    def has(self, scope: str) -> bool:
+        return scope in {
+            "addresses:address_book:read",
+            "addresses:address_book:write",
+            "addresses:contact:read",
+            "addresses:contact:write",
+            "addresses:contact:delete",
+            "addresses:sync:read",
+            "addresses:sync:write",
+        }
+
+
+class FakeLdapClient:
+    def __init__(self, entries: list[AddressLdapEntry], *, complete: bool = True) -> None:
+        self.entries = entries
+        self.complete = complete
+
+    def search(self, *, base_dn: str, search_filter: str, attributes: tuple[str, ...], page_size: int, max_entries: int) -> AddressLdapSearchResult:
+        del search_filter, attributes, max_entries
+        return AddressLdapSearchResult(
+            base_dn=base_dn,
+            entries=tuple(self.entries),
+            complete=self.complete,
+            page_size=page_size,
+        )
+
+
+def ldap_entry(
+    key: str,
+    *,
+    revision: str = "20260802090000Z",
+    organization: str = "Analysis Office",
+) -> AddressLdapEntry:
+    return AddressLdapEntry(
+        dn=f"uid={key},ou=people,dc=example,dc=test",
+        attributes={
+            "entryUUID": key,
+            "modifyTimestamp": revision,
+            "displayName": "Ada Lovelace",
+            "givenName": "Ada",
+            "sn": "Lovelace",
+            "mail": "ada@example.test",
+            "o": organization,
+            "memberOf": ["cn=analysts,ou=groups,dc=example,dc=test"],
+        },
+    )
+
+
+class AddressLdapSyncTests(unittest.TestCase):
+    def setUp(self) -> None:
+        engine = create_engine("sqlite+pysqlite:///:memory:")
+        Base.metadata.create_all(engine)
+        self.session = sessionmaker(bind=engine, expire_on_commit=False)()
+        self.principal = Principal()
+        self.book = AddressBook(
+            tenant_id="tenant-1",
+            scope_type="tenant",
+            scope_id="tenant-1",
+            name="Directory",
+            source_kind="local",
+            read_only=False,
+        )
+        self.session.add(self.book)
+        self.session.flush()
+        self.source = create_ldap_sync_source(
+            self.session,
+            self.principal,
+            self.book.id,
+            AddressLdapSourceCreateRequest(
+                url="ldaps://directory.example.test",
+                display_name="Corporate directory",
+                base_dn="ou=people,dc=example,dc=test",
+            ),
+        )
+        self.session.flush()
+
+    def test_preview_is_nonmutating_and_full_sync_is_idempotent(self) -> None:
+        client = FakeLdapClient([ldap_entry("person-1")])
+        preview = preview_sync_source(self.session, self.principal, self.source.id, client=client)
+        self.assertEqual(1, preview.stats.created)
+        self.assertEqual(0, self.session.query(Contact).count())
+
+        first = run_sync_source(self.session, self.principal, self.source.id, client=client)
+        self.assertEqual(1, first.stats.created)
+        contact = self.session.query(Contact).one()
+        self.assertEqual("ldap", contact.source_kind)
+        self.assertEqual("person-1", contact.provenance["ldap"]["source_key"])
+        self.assertEqual("succeeded", self.source.status)
+
+        repeated = run_sync_source(self.session, self.principal, self.source.id, client=client)
+        self.assertEqual(1, repeated.stats.unchanged)
+        self.assertEqual(1, self.session.query(Contact).count())
+
+        changed_client = FakeLdapClient(
+            [ldap_entry("person-1", revision="20260802100000Z", organization="Computing Office")]
+        )
+        changed = run_sync_source(self.session, self.principal, self.source.id, client=changed_client)
+        self.assertEqual(1, changed.stats.updated)
+        self.assertEqual("Computing Office", self.session.query(Contact).one().organization)
+
+    def test_only_complete_scans_plan_authoritative_deletes(self) -> None:
+        run_sync_source(
+            self.session,
+            self.principal,
+            self.source.id,
+            client=FakeLdapClient([ldap_entry("person-1")]),
+        )
+        incomplete = preview_sync_source(
+            self.session,
+            self.principal,
+            self.source.id,
+            client=FakeLdapClient([], complete=False),
+        )
+        self.assertEqual(0, incomplete.stats.deleted)
+        self.assertEqual(1, incomplete.stats.errors)
+        self.assertIsNone(self.session.query(Contact).one().deleted_at)
+
+        complete = run_sync_source(
+            self.session,
+            self.principal,
+            self.source.id,
+            client=FakeLdapClient([], complete=True),
+        )
+        self.assertEqual(1, complete.stats.deleted)
+        self.assertIsNotNone(self.session.query(Contact).one().deleted_at)
+
+    def test_connector_requires_encrypted_transport(self) -> None:
+        with self.assertRaisesRegex(AddressLdapError, "require StartTLS"):
+            AddressLdapClient(url="ldap://directory.example.test", start_tls=False)
+        with self.assertRaisesRegex(AddressLdapError, "must not contain credentials"):
+            AddressLdapClient(url="ldaps://user:secret@directory.example.test")
+
+    def test_source_is_always_read_only(self) -> None:
+        source = self.session.get(AddressSyncSource, self.source.id)
+        self.assertTrue(source.read_only)
+        self.assertEqual("read_only", source.sync_direction)
+        self.assertTrue(source.address_book.read_only)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from alembic.runtime.migration import MigrationContext
+from sqlalchemy import create_engine, inspect
+
+from govoplan_addresses.backend.manifest import get_manifest
+from govoplan_core.db.migrations import migrate_database
+
+
+class AddressesMigrationTests(unittest.TestCase):
+    def test_fresh_database_reaches_import_profile_head(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-addresses-migration-") as directory:
+            url = f"sqlite:///{Path(directory) / 'addresses.db'}"
+            migrate_database(
+                database_url=url,
+                enabled_modules=("addresses",),
+                manifest_factories=(get_manifest,),
+            )
+            engine = create_engine(url)
+            try:
+                with engine.connect() as connection:
+                    self.assertIn(
+                        "c5d7e8f9a0b1",
+                        set(MigrationContext.configure(connection).get_current_heads()),
+                    )
+                    tables = set(inspect(connection).get_table_names())
+                    self.assertIn("addresses_import_profiles", tables)
+                    self.assertIn("addresses_import_runs", tables)
+            finally:
+                engine.dispose()
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -15,7 +15,9 @@ from govoplan_addresses.backend.db.models import (
 from govoplan_addresses.backend.manifest import manifest
 from govoplan_addresses.backend.provider_state import (
    CARDDAV_PROVIDER_ID,
+    LDAP_PROVIDER_ID,
    carddav_provider_states,
+    ldap_provider_states,
 )
 from govoplan_core.core.provider_governance import ExternalProviderStateContext
 from govoplan_core.db.base import Base
@@ -103,6 +105,28 @@ class AddressesProviderStateTests(unittest.TestCase):
            CARDDAV_PROVIDER_ID,
            manifest.external_provider_state_providers[0].provider_id,
        )
+        self.assertEqual(LDAP_PROVIDER_ID, manifest.external_providers[1].id)
+        self.assertEqual(
+            LDAP_PROVIDER_ID,
+            manifest.external_provider_state_providers[1].provider_id,
+        )
+
+    def test_failed_ldap_source_is_stale_without_exposing_endpoint(self) -> None:
+        self.source.connector_type = "ldap"
+        self.source.display_name = "Directory"
+        self.source.status = "failed"
+        self.source.last_error = "connection failed"
+        self.session.flush()
+
+        state = ldap_provider_states(
+            ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
+        )[0]
+
+        self.assertEqual(LDAP_PROVIDER_ID, state.provider_id)
+        self.assertEqual("error", state.health)
+        self.assertEqual("current", state.freshness)
+        self.assertEqual("external_authoritative", state.authority_mode)
+        self.assertNotIn("dav.example.test", str(state.to_dict()))


 if __name__ == "__main__":
@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import base64
+from io import BytesIO
+import unittest
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from openpyxl import Workbook
+
+from govoplan_addresses.backend.db.models import AddressBook, Contact
+from govoplan_addresses.backend.import_schemas import (
+    AddressImportConfiguration,
+    AddressImportPreviewRequest,
+    AddressImportProfileCreateRequest,
+    AddressImportProfileUpdateRequest,
+    AddressImportRollbackRequest,
+)
+from govoplan_addresses.backend.imports import (
+    apply_address_import,
+    create_import_profile,
+    import_run_payload,
+    preview_address_import,
+    rollback_address_import,
+    update_import_profile,
+)
+from govoplan_core.db.base import Base
+
+
+class Principal:
+    account_id = "account-1"
+    group_ids = frozenset({"group-1"})
+
+    @property
+    def tenant_id(self) -> str:
+        return "tenant-1"
+
+    def has(self, scope: str) -> bool:
+        return scope in {
+            "addresses:address_book:read",
+            "addresses:address_book:write",
+            "addresses:contact:read",
+            "addresses:contact:write",
+        }
+
+
+def encoded(value: str) -> str:
+    return base64.b64encode(value.encode()).decode()
+
+
+class AddressTabularImportTests(unittest.TestCase):
+    def setUp(self) -> None:
+        engine = create_engine("sqlite+pysqlite:///:memory:")
+        Base.metadata.create_all(engine)
+        self.session = sessionmaker(bind=engine, expire_on_commit=False)()
+        self.principal = Principal()
+        self.book = AddressBook(
+            tenant_id="tenant-1",
+            scope_type="tenant",
+            scope_id="tenant-1",
+            name="Imported contacts",
+            source_kind="local",
+            read_only=False,
+        )
+        self.session.add(self.book)
+        self.profile = create_import_profile(
+            self.session,
+            self.principal,
+            AddressImportProfileCreateRequest(
+                scope_type="tenant",
+                name="Monthly contacts",
+                source_format="csv",
+                configuration=AddressImportConfiguration(
+                    delimiter=";",
+                    field_mappings={
+                        "source_key": "id",
+                        "given_name": "first",
+                        "family_name": "last",
+                        "email": "email",
+                        "organization": "organization",
+                    },
+                ),
+            ),
+        )
+        self.session.flush()
+
+    def test_preview_apply_repeat_and_guarded_rollback(self) -> None:
+        payload = AddressImportPreviewRequest(
+            profile_id=self.profile.id,
+            filename="contacts.csv",
+            content_base64=encoded(
+                "id;first;last;email;organization\n"
+                "1;Ada;Lovelace;ada@example.test;Analysis Office\n"
+                "2;Grace;Hopper;grace@example.test;Computing Office\n"
+            ),
+        )
+        run = preview_address_import(self.session, self.principal, self.book.id, payload)
+        self.assertEqual(2, run.statistics["create"])
+        self.assertFalse(run.diagnostics)
+
+        applied = apply_address_import(
+            self.session,
+            self.principal,
+            run.id,
+            expected_plan_hash=run.plan_hash,
+        )
+        self.assertEqual("applied", applied.status)
+        self.assertEqual(2, self.session.query(Contact).filter(Contact.deleted_at.is_(None)).count())
+        self.assertIs(applied, apply_address_import(self.session, self.principal, run.id, expected_plan_hash=run.plan_hash))
+        response_evidence = import_run_payload(applied)["result_evidence"]
+        self.assertEqual(2, response_evidence["created_contact_count"])
+        self.assertNotIn("created_contact_ids", response_evidence)
+        self.assertNotIn("updated_contacts", response_evidence)
+
+        repeated = preview_address_import(self.session, self.principal, self.book.id, payload)
+        self.assertEqual(2, repeated.statistics["unchanged"])
+        apply_address_import(self.session, self.principal, repeated.id, expected_plan_hash=repeated.plan_hash)
+        self.assertEqual(2, self.session.query(Contact).filter(Contact.deleted_at.is_(None)).count())
+
+        rolled_back = rollback_address_import(
+            self.session,
+            self.principal,
+            run.id,
+            AddressImportRollbackRequest(reason="The operator selected the wrong monthly file."),
+        )
+        self.assertEqual("rolled_back", rolled_back.status)
+        self.assertEqual(0, self.session.query(Contact).filter(Contact.deleted_at.is_(None)).count())
+
+    def test_duplicate_keys_and_changed_targets_block_apply(self) -> None:
+        duplicate = preview_address_import(
+            self.session,
+            self.principal,
+            self.book.id,
+            AddressImportPreviewRequest(
+                profile_id=self.profile.id,
+                filename="duplicates.csv",
+                content_base64=encoded(
+                    "id;first;last;email;organization\n"
+                    "1;Ada;Lovelace;ada@example.test;One\n"
+                    "1;Ada;Lovelace;ada@example.test;Two\n"
+                ),
+            ),
+        )
+        self.assertEqual(2, duplicate.statistics["conflict"])
+        with self.assertRaisesRegex(ValueError, "error diagnostics"):
+            apply_address_import(self.session, self.principal, duplicate.id, expected_plan_hash=duplicate.plan_hash)
+
+        initial = preview_address_import(
+            self.session,
+            self.principal,
+            self.book.id,
+            AddressImportPreviewRequest(
+                profile_id=self.profile.id,
+                filename="one.csv",
+                content_base64=encoded("id;first;last;email;organization\n1;Ada;Lovelace;ada@example.test;One\n"),
+            ),
+        )
+        apply_address_import(self.session, self.principal, initial.id, expected_plan_hash=initial.plan_hash)
+        changed = preview_address_import(
+            self.session,
+            self.principal,
+            self.book.id,
+            AddressImportPreviewRequest(
+                profile_id=self.profile.id,
+                filename="one.csv",
+                content_base64=encoded("id;first;last;email;organization\n1;Ada;Lovelace;ada@example.test;Two\n"),
+            ),
+        )
+        contact = self.session.query(Contact).filter(Contact.deleted_at.is_(None)).one()
+        contact.organization = "Concurrent edit"
+        self.session.flush()
+        with self.assertRaisesRegex(ValueError, "changed after preview"):
+            apply_address_import(self.session, self.principal, changed.id, expected_plan_hash=changed.plan_hash)
+
+    def test_profile_updates_create_immutable_versions(self) -> None:
+        next_profile = update_import_profile(
+            self.session,
+            self.principal,
+            self.profile.id,
+            payload=AddressImportProfileUpdateRequest(name="Monthly contacts v2"),
+        )
+        self.assertFalse(self.profile.is_current)
+        self.assertTrue(next_profile.is_current)
+        self.assertEqual(self.profile.profile_key, next_profile.profile_key)
+        self.assertEqual(2, next_profile.version)
+
+    def test_xlsx_sheet_selection_and_formula_rejection(self) -> None:
+        workbook = Workbook()
+        workbook.active.title = "Ignore"
+        sheet = workbook.create_sheet("Contacts")
+        sheet.append(["id", "first", "last", "email", "organization"])
+        sheet.append(["1", "Ada", "Lovelace", "ada@example.test", "Analysis Office"])
+        content = BytesIO()
+        workbook.save(content)
+        xlsx_profile = create_import_profile(
+            self.session,
+            self.principal,
+            AddressImportProfileCreateRequest(
+                scope_type="tenant",
+                name="Workbook contacts",
+                source_format="xlsx",
+                configuration=AddressImportConfiguration(
+                    sheet_name="Contacts",
+                    field_mappings={
+                        "source_key": "id",
+                        "given_name": "first",
+                        "family_name": "last",
+                        "email": "email",
+                        "organization": "organization",
+                    },
+                ),
+            ),
+        )
+        self.session.flush()
+        run = preview_address_import(
+            self.session,
+            self.principal,
+            self.book.id,
+            AddressImportPreviewRequest(
+                profile_id=xlsx_profile.id,
+                filename="contacts.xlsx",
+                content_base64=base64.b64encode(content.getvalue()).decode(),
+            ),
+        )
+        self.assertEqual(1, run.statistics["create"])
+
+        sheet["E2"] = "=CONCAT(\"Analysis\", \" Office\")"
+        content = BytesIO()
+        workbook.save(content)
+        with self.assertRaisesRegex(ValueError, "formulas are never evaluated"):
+            preview_address_import(
+                self.session,
+                self.principal,
+                self.book.id,
+                AddressImportPreviewRequest(
+                    profile_id=xlsx_profile.id,
+                    filename="contacts.xlsx",
+                    content_base64=base64.b64encode(content.getvalue()).decode(),
+                ),
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()