From 8740fb33f8a2fb873944057a9d3246e597b64ef1 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Fri, 21 Aug 2026 01:19:17 +0200 Subject: [PATCH] feat(addresses): add governed DSAR coverage --- README.md | 11 + .../backend/dsar_provider.py | 1155 +++++++++++++++++ src/govoplan_addresses/backend/manifest.py | 449 ++++++- tests/test_dsar_provider.py | 588 +++++++++ 4 files changed, 2141 insertions(+), 62 deletions(-) create mode 100644 src/govoplan_addresses/backend/dsar_provider.py create mode 100644 tests/test_dsar_provider.py diff --git a/README.md b/README.md index f39145e..4b0beaa 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,17 @@ The module exposes core-mediated capabilities for: pickers. - `distribution.recipient_channel_facts`: current channel, governance, and quality facts for distribution and Policy consumers. +- `privacy.dsar.addresses`: tenant-bounded, minimized data-subject discovery + across contacts, contact points, list use, governance, provenance, + synchronization evidence, and operator attribution. + +The DSAR provider accepts corroborated email/account selectors and namespaced +Addresses references. It does not export connector state, raw import or sync +payloads, opaque metadata, snapshot payloads, or merge before/after payloads. +Reusable contacts are never deleted automatically: shared/synchronized contact +changes require an authorized dependency review through the ordinary Addresses +workflows, while governance, quality, merge, sync, import, and attribution +evidence is retained with an explicit reason. `addresses.recipient_source` returns: diff --git a/src/govoplan_addresses/backend/dsar_provider.py b/src/govoplan_addresses/backend/dsar_provider.py new file mode 100644 index 0000000..c2d57e1 --- /dev/null +++ b/src/govoplan_addresses/backend/dsar_provider.py @@ -0,0 +1,1155 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone + +from sqlalchemy import func, or_ +from sqlalchemy.orm import Session + +from govoplan_addresses.backend.db.models import ( + AddressBook, + AddressImportProfile, + AddressImportRun, + AddressList, + AddressListEntry, + AddressSyncConflict, + AddressSyncSource, + AddressSyncTombstone, + Contact, + ContactChannelRule, + ContactEmail, + ContactFieldProvenance, + ContactMergeRecord, + ContactPhone, + ContactPointQualityDecision, + ContactPointSnapshot, + ContactPostalAddress, + ContactRedirect, +) +from govoplan_core.core.dsar import ( + DsarErasureActionRef, + DsarExecutionResultRef, + DsarRecordRef, + DsarSubjectRef, + dsar_capability_name, +) + + +ADDRESSES_DSAR_CAPABILITY = dsar_capability_name("addresses") +_MAX_RECORDS = 5_000 + + +@dataclass(frozen=True, slots=True) +class _SubjectSelectors: + account_id: str | None + email: str | None + references: Mapping[str, str] + + +class AddressesDsarProvider: + provider_id = "addresses" + module_id = "addresses" + + def search_subject( + self, + session: object, + *, + tenant_id: str, + subject: DsarSubjectRef, + ) -> Sequence[DsarRecordRef]: + db = _session(session) + selectors = _subject_selectors(subject) + if selectors is None: + return () + + contact_ids = _subject_contact_ids( + db, + tenant_id=tenant_id, + selectors=selectors, + ) + if contact_ids is None: + return () + + records: list[DsarRecordRef] = [] + seen: set[tuple[str, str]] = set() + + def append(record: DsarRecordRef) -> None: + key = (record.resource_type, record.resource_id) + if key in seen: + return + if len(records) >= _MAX_RECORDS: + raise ValueError( + "Addresses DSAR result limit exceeded; narrow the subject selectors." + ) + seen.add(key) + records.append(record) + + contacts = _rows_by_ids( + db, + Contact, + tenant_id=tenant_id, + ids=contact_ids, + ) + for contact in contacts: + match_fields = [] + if selectors.references.get("contact") == contact.id: + match_fields.append("reference") + if selectors.email and any( + _normalized_email(item.email) == selectors.email + or _normalized_email(item.normalized_email) == selectors.email + for item in contact.emails + ): + match_fields.append("email") + append( + _record( + "addresses_contact", + contact.id, + "contact_identity", + f"Address contact {contact.display_name}", + { + "match_fields": match_fields, + "display_name": _bounded_text(contact.display_name, 255), + "given_name": _bounded_text(contact.given_name, 255), + "family_name": _bounded_text(contact.family_name, 255), + "organization": _bounded_text(contact.organization, 255), + "role_title": _bounded_text(contact.role_title, 255), + "note": _bounded_text(contact.note, 2_000), + "tags": [str(value)[:120] for value in contact.tags[:100]], + "source_kind": contact.source_kind, + "source_revision": _bounded_text( + contact.source_revision, + 255, + ), + "deleted_at": _iso(contact.deleted_at), + }, + observed_at=contact.updated_at, + ) + ) + + for email in _contact_children( + db, + ContactEmail, + tenant_id=tenant_id, + contact_ids=contact_ids, + ): + append( + _record( + "addresses_contact_email", + email.id, + "electronic_contact_point", + "Address email contact point", + { + "contact_id": email.contact_id, + "match_fields": _contact_point_match_fields( + email, + selectors=selectors, + reference_kind="contact_email", + ), + "label": _bounded_text(email.label, 80), + "email": _bounded_text(email.email, 320), + "original_email": _bounded_text(email.original_email, 320), + "normalized_email": _bounded_text( + email.normalized_email, + 320, + ), + "is_primary": email.is_primary, + "order_index": email.order_index, + }, + observed_at=email.updated_at, + ) + ) + + for phone in _contact_children( + db, + ContactPhone, + tenant_id=tenant_id, + contact_ids=contact_ids, + ): + append( + _record( + "addresses_contact_phone", + phone.id, + "telephone_contact_point", + "Address telephone contact point", + { + "contact_id": phone.contact_id, + "match_fields": _contact_point_match_fields( + phone, + selectors=selectors, + reference_kind="contact_phone", + ), + "label": _bounded_text(phone.label, 80), + "phone": _bounded_text(phone.phone, 100), + "original_phone": _bounded_text(phone.original_phone, 100), + "normalized_phone": _bounded_text( + phone.normalized_phone, + 100, + ), + "is_primary": phone.is_primary, + "order_index": phone.order_index, + }, + observed_at=phone.updated_at, + ) + ) + + for postal in _contact_children( + db, + ContactPostalAddress, + tenant_id=tenant_id, + contact_ids=contact_ids, + ): + append( + _record( + "addresses_contact_postal_address", + postal.id, + "postal_contact_point", + "Address postal contact point", + { + "contact_id": postal.contact_id, + "match_fields": _contact_point_match_fields( + postal, + selectors=selectors, + reference_kind="contact_postal_address", + ), + "label": _bounded_text(postal.label, 80), + "street": _bounded_text(postal.street, 500), + "postal_code": _bounded_text(postal.postal_code, 40), + "locality": _bounded_text(postal.locality, 255), + "region": _bounded_text(postal.region, 255), + "country": _bounded_text(postal.country, 255), + "is_primary": postal.is_primary, + "order_index": postal.order_index, + }, + observed_at=postal.updated_at, + ) + ) + + for entry in _list_entries( + db, + tenant_id=tenant_id, + contact_ids=contact_ids, + ): + append( + _record( + "addresses_list_membership", + entry.id, + "address_list_use", + "Address-list membership", + { + "contact_id": entry.contact_id, + "address_list_id": entry.address_list_id, + "address_list_name": _bounded_text( + entry.address_list.name, + 255, + ), + "target_kind": entry.target_kind, + "contact_email_id": entry.contact_email_id, + "contact_postal_address_id": (entry.contact_postal_address_id), + "label": _bounded_text(entry.label, 255), + "order_index": entry.order_index, + }, + observed_at=entry.updated_at, + ) + ) + + for rule in _related_or_attributed( + db, + ContactChannelRule, + tenant_id=tenant_id, + related_field=ContactChannelRule.contact_id, + related_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=(ContactChannelRule.created_by_account_id,), + ): + append( + _record( + "addresses_channel_rule", + rule.id, + "communication_governance_evidence", + "Address communication-governance decision", + { + "match_fields": _related_actor_match_fields( + rule, + contact_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=("created_by_account_id",), + ), + "contact_id": ( + rule.contact_id if rule.contact_id in contact_ids else None + ), + "channel": rule.channel, + "purpose": _bounded_text(rule.purpose, 120), + "contact_point_id": ( + rule.contact_point_id + if rule.contact_id in contact_ids + else None + ), + "decision": rule.decision, + "legal_basis": _bounded_text(rule.legal_basis, 255), + "reason": _bounded_text(rule.reason, 2_000), + "preference_rank": rule.preference_rank, + "locale": _bounded_text(rule.locale, 20), + "effective_from": _iso(rule.effective_from), + "effective_until": _iso(rule.effective_until), + }, + observed_at=rule.updated_at, + immutable=True, + retention_reason=( + "Communication-governance decisions are effective-dated " + "institutional evidence and must be corrected by a new decision." + ), + ) + ) + + for decision in _related_or_attributed( + db, + ContactPointQualityDecision, + tenant_id=tenant_id, + related_field=ContactPointQualityDecision.contact_id, + related_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=(ContactPointQualityDecision.created_by_account_id,), + ): + append( + _record( + "addresses_quality_decision", + decision.id, + "contact_quality_evidence", + "Address contact-point quality decision", + { + "match_fields": _related_actor_match_fields( + decision, + contact_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=("created_by_account_id",), + ), + "contact_id": ( + decision.contact_id + if decision.contact_id in contact_ids + else None + ), + "channel": decision.channel, + "contact_point_id": ( + decision.contact_point_id + if decision.contact_id in contact_ids + else None + ), + "state": decision.state, + "reason_code": decision.reason_code, + "reason": _bounded_text(decision.reason, 2_000), + "effective_from": _iso(decision.effective_from), + "effective_until": _iso(decision.effective_until), + }, + observed_at=decision.updated_at, + immutable=True, + retention_reason=( + "Quality decisions explain recipient eligibility and are " + "retained as effective-dated evidence." + ), + ) + ) + + for provenance in _related_or_attributed( + db, + ContactFieldProvenance, + tenant_id=tenant_id, + related_field=ContactFieldProvenance.contact_id, + related_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=(ContactFieldProvenance.created_by_account_id,), + ): + append( + _record( + "addresses_field_provenance", + provenance.id, + "contact_provenance_evidence", + "Address field provenance", + { + "match_fields": _related_actor_match_fields( + provenance, + contact_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=("created_by_account_id",), + ), + "contact_id": ( + provenance.contact_id + if provenance.contact_id in contact_ids + else None + ), + "field_path": _bounded_text(provenance.field_path, 255), + "source_kind": provenance.source_kind, + "source_revision": _bounded_text( + provenance.source_revision, + 255, + ), + "precedence": provenance.precedence, + "selected": provenance.selected, + "reason_code": provenance.reason_code, + "explanation": _bounded_text( + provenance.explanation, + 2_000, + ), + "visibility": provenance.visibility, + }, + observed_at=provenance.updated_at, + immutable=True, + retention_reason=( + "Field provenance is retained to explain source authority, " + "normalization, and merge survivorship." + ), + ) + ) + + for merge in _matching_merges( + db, + tenant_id=tenant_id, + contact_ids=contact_ids, + account_id=selectors.account_id, + ): + related_ids = { + merge.winner_contact_id, + *(str(item) for item in merge.loser_contact_ids), + } & contact_ids + actor_fields = _actor_match_fields( + merge, + selectors.account_id, + ( + "created_by_account_id", + "recovered_by_account_id", + ), + ) + append( + _record( + "addresses_merge_record", + merge.id, + "contact_merge_evidence", + "Address contact merge record", + { + "match_fields": ( + (["contact_id"] if related_ids else []) + actor_fields + ), + "related_contact_ids": sorted(related_ids), + "status": merge.status, + "reason": _bounded_text(merge.reason, 2_000), + "before_hash": merge.before_hash, + "after_hash": merge.after_hash, + "recovered_at": _iso(merge.recovered_at), + "recovery_action": merge.recovery_action, + "recovery_reason": _bounded_text( + merge.recovery_reason, + 2_000, + ), + }, + observed_at=merge.updated_at, + immutable=True, + retention_reason=( + "Merge records and hashes are retained so redirects, list " + "repairs, undo, and split operations remain explainable." + ), + ) + ) + + for redirect in _matching_redirects( + db, + tenant_id=tenant_id, + contact_ids=contact_ids, + ): + append( + _record( + "addresses_contact_redirect", + redirect.id, + "contact_merge_evidence", + "Address contact redirect", + { + "source_contact_id": redirect.source_contact_id, + "target_contact_id": redirect.target_contact_id, + "merge_record_id": redirect.merge_record_id, + "ended_at": _iso(redirect.ended_at), + }, + observed_at=redirect.updated_at, + immutable=True, + retention_reason=( + "Contact redirects preserve resolution of historical references." + ), + ) + ) + + for tombstone in _related_rows( + db, + AddressSyncTombstone, + tenant_id=tenant_id, + related_field=AddressSyncTombstone.contact_id, + related_ids=contact_ids, + ): + append( + _record( + "addresses_sync_tombstone", + tombstone.id, + "address_sync_evidence", + "Address synchronization tombstone", + { + "contact_id": tombstone.contact_id, + "sync_source_id": tombstone.sync_source_id, + "address_book_id": tombstone.address_book_id, + "local_deleted_at": _iso(tombstone.local_deleted_at), + "remote_deleted_at": _iso(tombstone.remote_deleted_at), + "synced_at": _iso(tombstone.synced_at), + }, + observed_at=tombstone.updated_at, + immutable=True, + retention_reason=( + "Synchronization tombstones prevent accidental recreation " + "and document source lifecycle outcomes." + ), + ) + ) + + for conflict in _related_or_attributed( + db, + AddressSyncConflict, + tenant_id=tenant_id, + related_field=AddressSyncConflict.contact_id, + related_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=(AddressSyncConflict.resolved_by_account_id,), + ): + append( + _record( + "addresses_sync_conflict", + conflict.id, + "address_sync_evidence", + "Address synchronization conflict", + { + "match_fields": _related_actor_match_fields( + conflict, + contact_ids=contact_ids, + account_id=selectors.account_id, + actor_fields=("resolved_by_account_id",), + ), + "contact_id": ( + conflict.contact_id + if conflict.contact_id in contact_ids + else None + ), + "sync_source_id": conflict.sync_source_id, + "address_book_id": conflict.address_book_id, + "field_path": _bounded_text(conflict.field_path, 500), + "status": conflict.status, + "resolution": conflict.resolution, + "resolved_at": _iso(conflict.resolved_at), + }, + observed_at=conflict.updated_at, + immutable=True, + retention_reason=( + "Synchronization conflict outcomes are retained to explain " + "which authoritative value was selected." + ), + ) + ) + + _append_actor_attribution_records( + db, + tenant_id=tenant_id, + account_id=selectors.account_id, + append=append, + ) + + return tuple(records) + + def plan_erasure( + self, + session: object, + *, + tenant_id: str, + subject: DsarSubjectRef, + records: Sequence[DsarRecordRef], + ) -> Sequence[DsarErasureActionRef]: + del tenant_id + _session(session) + if _subject_selectors(subject) is None: + raise ValueError("Addresses DSAR subject selectors conflict.") + actions: list[DsarErasureActionRef] = [] + for record in records: + _validate_record(record) + if record.immutable_evidence: + kind = "retain" + title = f"Retain {record.title}" + rationale = record.retention_reason or ( + "Addresses governance evidence must be retained." + ) + else: + kind = "manual_review" + title = f"Review {record.title}" + rationale = ( + "Reusable address data can be shared, synchronized, merged, " + "or referenced by immutable recipient evidence. An authorized " + "operator must correct, archive, or erase it through the owning " + "Addresses workflow after reviewing those dependencies." + ) + actions.append( + DsarErasureActionRef( + action_id=( + f"addresses:{kind}:{record.resource_type}:{record.resource_id}" + ), + provider_id=self.provider_id, + module_id=self.module_id, + kind=kind, + resource_type=record.resource_type, + resource_id=record.resource_id, + title=title, + rationale=rationale, + executable=False, + ) + ) + return tuple(actions) + + def execute_erasure( + self, + session: object, + *, + tenant_id: str, + subject: DsarSubjectRef, + actions: Sequence[DsarErasureActionRef], + request_id: str, + ) -> Sequence[DsarExecutionResultRef]: + del tenant_id + _session(session) + if _subject_selectors(subject) is None: + raise ValueError("Addresses DSAR subject selectors conflict.") + results: list[DsarExecutionResultRef] = [] + for action in actions: + _validate_action(action) + if action.executable: + raise ValueError( + "Addresses DSAR does not publish executable erasure actions." + ) + results.append( + DsarExecutionResultRef( + action_id=action.action_id, + status="blocked", + summary=( + "Use the authorized Addresses correction, archive, source, " + "merge, or governance workflow after dependency review." + ), + evidence={"request_id": request_id}, + ) + ) + return tuple(results) + + +def _subject_contact_ids( + session: Session, + *, + tenant_id: str, + selectors: _SubjectSelectors, +) -> set[str] | None: + email_ids: set[str] = set() + if selectors.email: + email_ids = { + row[0] + for row in _bounded_rows( + session.query(Contact.id) + .join(ContactEmail, ContactEmail.contact_id == Contact.id) + .filter( + Contact.tenant_id == tenant_id, + or_( + func.lower(ContactEmail.email) == selectors.email, + func.lower(ContactEmail.normalized_email) == selectors.email, + ), + ) + .order_by(Contact.id) + ) + } + + direct_ids: set[str] = set() + direct_queries = ( + ("contact", Contact, Contact.id), + ("contact_email", ContactEmail, ContactEmail.id), + ("contact_phone", ContactPhone, ContactPhone.id), + ( + "contact_postal_address", + ContactPostalAddress, + ContactPostalAddress.id, + ), + ) + for kind, model, id_field in direct_queries: + reference = selectors.references.get(kind) + if not reference: + continue + query = session.query(Contact.id) + if model is not Contact: + query = query.join(model, model.contact_id == Contact.id) + row = ( + query.filter( + Contact.tenant_id == tenant_id, + id_field == reference, + ) + .order_by(Contact.id) + .one_or_none() + ) + if row is None: + return None + direct_ids.add(row[0]) + + if selectors.email and direct_ids and not direct_ids.issubset(email_ids): + return None + return email_ids | direct_ids + + +def _append_actor_attribution_records( + session: Session, + *, + tenant_id: str, + account_id: str | None, + append: object, +) -> None: + if not account_id: + return + + definitions = ( + ( + AddressBook, + "addresses_address_book_attribution", + "address_book_configuration", + ("created_by_account_id", "updated_by_account_id"), + lambda row: { + "scope_type": row.scope_type, + "scope_id": row.scope_id, + "name": _bounded_text(row.name, 255), + "source_kind": row.source_kind, + "read_only": row.read_only, + "deleted_at": _iso(row.deleted_at), + }, + ), + ( + Contact, + "addresses_contact_attribution", + "contact_configuration", + ("created_by_account_id", "updated_by_account_id"), + lambda row: { + "address_book_id": row.address_book_id, + "source_kind": row.source_kind, + "deleted_at": _iso(row.deleted_at), + }, + ), + ( + AddressList, + "addresses_list_attribution", + "address_list_configuration", + ("created_by_account_id", "updated_by_account_id"), + lambda row: { + "address_book_id": row.address_book_id, + "name": _bounded_text(row.name, 255), + "source_kind": row.source_kind, + "read_only": row.read_only, + "deleted_at": _iso(row.deleted_at), + }, + ), + ( + AddressSyncSource, + "addresses_sync_source_attribution", + "address_source_configuration", + ("created_by_account_id", "updated_by_account_id"), + lambda row: { + "address_book_id": row.address_book_id, + "connector_type": row.connector_type, + "display_name": _bounded_text(row.display_name, 255), + "sync_direction": row.sync_direction, + "read_only": row.read_only, + "enabled": row.enabled, + "status": row.status, + }, + ), + ( + AddressImportProfile, + "addresses_import_profile_attribution", + "address_import_configuration", + ("created_by_account_id",), + lambda row: { + "profile_key": row.profile_key, + "version": row.version, + "scope_type": row.scope_type, + "scope_id": row.scope_id, + "name": _bounded_text(row.name, 255), + "source_format": row.source_format, + "is_current": row.is_current, + "superseded_at": _iso(row.superseded_at), + }, + ), + ( + AddressImportRun, + "addresses_import_run_attribution", + "address_import_evidence", + ("created_by_account_id",), + lambda row: { + "address_book_id": row.address_book_id, + "profile_id": row.profile_id, + "source_filename": _bounded_text(row.source_filename, 500), + "source_format": row.source_format, + "input_hash": row.input_hash, + "plan_hash": row.plan_hash, + "status": row.status, + "row_count": row.row_count, + "applied_at": _iso(row.applied_at), + "rolled_back_at": _iso(row.rolled_back_at), + }, + ), + ( + ContactPointSnapshot, + "addresses_snapshot_attribution", + "recipient_snapshot_evidence", + ("created_by_account_id",), + lambda row: { + "source_id": row.source_id, + "contract_version": row.contract_version, + "source_revision": row.source_revision, + "purpose": _bounded_text(row.purpose, 120), + "effective_at": _iso(row.effective_at), + "generated_at": _iso(row.generated_at), + "recipient_count": row.recipient_count, + "excluded_count": row.excluded_count, + "snapshot_hash": row.snapshot_hash, + }, + ), + ) + append_record = append + for model, resource_type, category, actor_fields, data_factory in definitions: + conditions = [getattr(model, field) == account_id for field in actor_fields] + rows = _bounded_rows( + session.query(model) + .filter(model.tenant_id == tenant_id, or_(*conditions)) + .order_by(model.id) + ) + for row in rows: + data = data_factory(row) + data["match_fields"] = _actor_match_fields( + row, + account_id, + actor_fields, + ) + append_record( # type: ignore[operator] + _record( + resource_type, + row.id, + category, + "Addresses operator attribution", + data, + observed_at=row.updated_at, + immutable=True, + retention_reason=( + "Operator attribution is retained with the governed " + "configuration or evidence record for accountability." + ), + ) + ) + + +def _subject_selectors(subject: DsarSubjectRef) -> _SubjectSelectors | None: + groups = { + "account_id": ( + subject.account_id, + subject.external_references.get("addresses.account"), + subject.external_references.get("access.account"), + ), + "email": ( + subject.email, + subject.external_references.get("addresses.email"), + ), + } + normalized: dict[str, str | None] = {} + for key, values in groups.items(): + distinct = { + value + for item in values + if ( + value := ( + _normalized_email(item) if key == "email" else _normalized_id(item) + ) + ) + } + if len(distinct) > 1: + return None + normalized[key] = next(iter(distinct), None) + + aliases = { + "addresses.contact": "contact", + "addresses.contact_email": "contact_email", + "addresses.contact_phone": "contact_phone", + "addresses.contact_postal_address": "contact_postal_address", + } + references = { + target: value + for source, target in aliases.items() + if (value := _normalized_id(subject.external_references.get(source))) + } + return _SubjectSelectors(references=references, **normalized) + + +def _rows_by_ids( + session: Session, + model: type, + *, + tenant_id: str, + ids: set[str], +) -> list[object]: + if not ids: + return [] + return _bounded_rows( + session.query(model) + .filter(model.tenant_id == tenant_id, model.id.in_(ids)) + .order_by(model.id) + ) + + +def _contact_children( + session: Session, + model: type, + *, + tenant_id: str, + contact_ids: set[str], +) -> list[object]: + if not contact_ids: + return [] + return _bounded_rows( + session.query(model) + .join(Contact, model.contact_id == Contact.id) + .filter( + Contact.tenant_id == tenant_id, + model.contact_id.in_(contact_ids), + ) + .order_by(model.id) + ) + + +def _list_entries( + session: Session, + *, + tenant_id: str, + contact_ids: set[str], +) -> list[AddressListEntry]: + if not contact_ids: + return [] + return _bounded_rows( + session.query(AddressListEntry) + .join(AddressList, AddressListEntry.address_list_id == AddressList.id) + .filter( + AddressList.tenant_id == tenant_id, + AddressListEntry.contact_id.in_(contact_ids), + ) + .order_by(AddressListEntry.id) + ) + + +def _related_rows( + session: Session, + model: type, + *, + tenant_id: str, + related_field: object, + related_ids: set[str], +) -> list[object]: + if not related_ids: + return [] + return _bounded_rows( + session.query(model) + .filter( + model.tenant_id == tenant_id, + related_field.in_(related_ids), # type: ignore[attr-defined] + ) + .order_by(model.id) + ) + + +def _related_or_attributed( + session: Session, + model: type, + *, + tenant_id: str, + related_field: object, + related_ids: set[str], + account_id: str | None, + actor_fields: Sequence[object], +) -> list[object]: + conditions = [] + if related_ids: + conditions.append(related_field.in_(related_ids)) # type: ignore[attr-defined] + if account_id: + conditions.extend(field == account_id for field in actor_fields) + if not conditions: + return [] + return _bounded_rows( + session.query(model) + .filter(model.tenant_id == tenant_id, or_(*conditions)) + .order_by(model.id) + ) + + +def _matching_merges( + session: Session, + *, + tenant_id: str, + contact_ids: set[str], + account_id: str | None, +) -> list[ContactMergeRecord]: + rows = _bounded_rows( + session.query(ContactMergeRecord) + .filter(ContactMergeRecord.tenant_id == tenant_id) + .order_by(ContactMergeRecord.id) + ) + return [ + row + for row in rows + if row.winner_contact_id in contact_ids + or bool({str(item) for item in row.loser_contact_ids} & contact_ids) + or bool( + _actor_match_fields( + row, + account_id, + ("created_by_account_id", "recovered_by_account_id"), + ) + ) + ] + + +def _matching_redirects( + session: Session, + *, + tenant_id: str, + contact_ids: set[str], +) -> list[ContactRedirect]: + if not contact_ids: + return [] + return _bounded_rows( + session.query(ContactRedirect) + .filter( + ContactRedirect.tenant_id == tenant_id, + or_( + ContactRedirect.source_contact_id.in_(contact_ids), + ContactRedirect.target_contact_id.in_(contact_ids), + ), + ) + .order_by(ContactRedirect.id) + ) + + +def _contact_point_match_fields( + row: object, + *, + selectors: _SubjectSelectors, + reference_kind: str, +) -> list[str]: + fields = [] + if selectors.references.get(reference_kind) == getattr(row, "id"): + fields.append("reference") + if reference_kind == "contact_email" and selectors.email: + if selectors.email in { + _normalized_email(getattr(row, "email", None)), + _normalized_email(getattr(row, "normalized_email", None)), + }: + fields.append("email") + return fields + + +def _related_actor_match_fields( + row: object, + *, + contact_ids: set[str], + account_id: str | None, + actor_fields: Sequence[str], +) -> list[str]: + fields = [] + if getattr(row, "contact_id", None) in contact_ids: + fields.append("contact_id") + fields.extend(_actor_match_fields(row, account_id, actor_fields)) + return fields + + +def _actor_match_fields( + row: object, + account_id: str | None, + fields: Sequence[str], +) -> list[str]: + if not account_id: + return [] + return [field for field in fields if getattr(row, field, None) == account_id] + + +def _validate_record(record: DsarRecordRef) -> None: + if record.provider_id != "addresses" or record.module_id != "addresses": + raise ValueError("Addresses DSAR received a foreign provider record.") + + +def _validate_action(action: DsarErasureActionRef) -> None: + if action.provider_id != "addresses" or action.module_id != "addresses": + raise ValueError("Addresses DSAR received a foreign provider action.") + + +def _record( + resource_type: str, + resource_id: str, + category: str, + title: str, + data: Mapping[str, object], + *, + observed_at: datetime | None = None, + immutable: bool = False, + retention_reason: str | None = None, +) -> DsarRecordRef: + return DsarRecordRef( + provider_id="addresses", + module_id="addresses", + resource_type=resource_type, + resource_id=resource_id, + category=category, + title=title, + data=data, + observed_at=observed_at, + immutable_evidence=immutable, + retention_reason=retention_reason, + source_path="/address-book", + ) + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Addresses DSAR provider requires a SQLAlchemy session.") + return value + + +def _bounded_rows(query: object) -> list[object]: + rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined] + if len(rows) > _MAX_RECORDS: + raise ValueError( + "Addresses DSAR match limit exceeded; narrow the subject selectors." + ) + return rows + + +def _bounded_text(value: str | None, limit: int) -> str | None: + return value[:limit] if value else None + + +def _normalized_email(value: object) -> str | None: + if not isinstance(value, str): + return None + value = value.strip().casefold() + return value or None + + +def _normalized_id(value: object) -> str | None: + if value is None: + return None + value = str(value).strip() + return value or None + + +def _iso(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.isoformat() + + +__all__ = ["ADDRESSES_DSAR_CAPABILITY", "AddressesDsarProvider"] diff --git a/src/govoplan_addresses/backend/manifest.py b/src/govoplan_addresses/backend/manifest.py index 8046e7a..b47d1cc 100644 --- a/src/govoplan_addresses/backend/manifest.py +++ b/src/govoplan_addresses/backend/manifest.py @@ -11,12 +11,21 @@ from govoplan_addresses.backend.capabilities import ( CAPABILITY_ADDRESSES_RECIPIENT_SOURCE, ) from govoplan_addresses.backend.db import models as addresses_models # noqa: F401 - populate address ORM metadata -from govoplan_core.core.access import CAPABILITY_AUTH_PERMISSION_EVALUATOR, CAPABILITY_AUTH_PRINCIPAL_RESOLVER -from govoplan_core.core.contact_points import CAPABILITY_ADDRESSES_CONTACT_POINT_RESOLUTION -from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard +from govoplan_core.core.access import ( + CAPABILITY_AUTH_PERMISSION_EVALUATOR, + CAPABILITY_AUTH_PRINCIPAL_RESOLVER, +) +from govoplan_core.core.contact_points import ( + CAPABILITY_ADDRESSES_CONTACT_POINT_RESOLUTION, +) +from govoplan_core.core.module_guards import ( + drop_table_retirement_provider, + persistent_table_uninstall_guard, +) from govoplan_core.core.people import CAPABILITY_ADDRESSES_PEOPLE_SEARCH from govoplan_core.core.distribution_lists import CAPABILITY_RECIPIENT_CHANNEL_FACTS from govoplan_core.core.modules import ( + CapabilityDocumentation, DocumentationTopic, FrontendModule, FrontendRoute, @@ -38,6 +47,7 @@ from govoplan_core.core.provider_governance import ( ) from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base +from govoplan_addresses.backend.dsar_provider import ADDRESSES_DSAR_CAPABILITY from govoplan_addresses.backend.provider_state import ( CARDDAV_PROVIDER_ID, LDAP_PROVIDER_ID, @@ -46,6 +56,13 @@ from govoplan_addresses.backend.provider_state import ( ) +def _addresses_dsar_provider(context: ModuleContext) -> object: + del context + from govoplan_addresses.backend.dsar_provider import AddressesDsarProvider + + return AddressesDsarProvider() + + _addresses_table_retirement_provider = drop_table_retirement_provider( addresses_models.AddressImportRun, addresses_models.AddressImportProfile, @@ -77,10 +94,18 @@ def _addresses_retirement_provider(session: object | None, module_id: str): return plan def executor(execute_session: object, execute_module_id: str) -> None: - if not hasattr(execute_session, "get_bind") or not hasattr(execute_session, "query"): - raise RuntimeError("No database session is available for Addresses credential retirement.") - if inspect(execute_session.get_bind()).has_table(addresses_models.AddressSyncSource.__tablename__): - from govoplan_addresses.backend.service import audit_address_credentials_for_retirement + if not hasattr(execute_session, "get_bind") or not hasattr( + execute_session, "query" + ): + raise RuntimeError( + "No database session is available for Addresses credential retirement." + ) + if inspect(execute_session.get_bind()).has_table( + addresses_models.AddressSyncSource.__tablename__ + ): + from govoplan_addresses.backend.service import ( + audit_address_credentials_for_retirement, + ) audit_address_credentials_for_retirement(execute_session) base_executor(execute_session, execute_module_id) @@ -110,21 +135,77 @@ def _permission(scope: str, label: str, description: str) -> PermissionDefinitio PERMISSIONS = ( - _permission("addresses:address_book:read", "View address books", "List address books visible to the current principal."), - _permission("addresses:address_book:write", "Manage address books", "Create and edit local address books."), - _permission("addresses:address_book:delete", "Delete address books", "Soft-delete local address books."), - _permission("addresses:address_book:admin", "Administer address books", "Manage system-scoped address books and future sync sources."), - _permission("addresses:address_list:read", "View address lists", "List reusable address lists and their entries."), - _permission("addresses:address_list:write", "Manage address lists", "Create and edit reusable address lists."), - _permission("addresses:address_list:delete", "Delete address lists", "Soft-delete reusable address lists."), - _permission("addresses:contact:read", "View contacts", "List and lookup contacts in visible address books."), - _permission("addresses:contact:write", "Manage contacts", "Create and edit local contacts."), - _permission("addresses:contact:delete", "Delete contacts", "Soft-delete local contacts."), - _permission("addresses:governance:read", "View communication governance", "Inspect effective-dated consent, suppression, and channel-preference facts."), - _permission("addresses:governance:write", "Manage communication governance", "Record and end consent, suppression, and channel-preference facts."), - _permission("addresses:sync:read", "View address sync", "Inspect address sync sources, conflicts, tombstones, and diagnostics."), - _permission("addresses:sync:write", "Manage address sync", "Bind address books to external sources and record sync state."), - _permission("addresses:sync:admin", "Administer address sync", "Administer address sync connectors and future destructive sync operations."), + _permission( + "addresses:address_book:read", + "View address books", + "List address books visible to the current principal.", + ), + _permission( + "addresses:address_book:write", + "Manage address books", + "Create and edit local address books.", + ), + _permission( + "addresses:address_book:delete", + "Delete address books", + "Soft-delete local address books.", + ), + _permission( + "addresses:address_book:admin", + "Administer address books", + "Manage system-scoped address books and future sync sources.", + ), + _permission( + "addresses:address_list:read", + "View address lists", + "List reusable address lists and their entries.", + ), + _permission( + "addresses:address_list:write", + "Manage address lists", + "Create and edit reusable address lists.", + ), + _permission( + "addresses:address_list:delete", + "Delete address lists", + "Soft-delete reusable address lists.", + ), + _permission( + "addresses:contact:read", + "View contacts", + "List and lookup contacts in visible address books.", + ), + _permission( + "addresses:contact:write", "Manage contacts", "Create and edit local contacts." + ), + _permission( + "addresses:contact:delete", "Delete contacts", "Soft-delete local contacts." + ), + _permission( + "addresses:governance:read", + "View communication governance", + "Inspect effective-dated consent, suppression, and channel-preference facts.", + ), + _permission( + "addresses:governance:write", + "Manage communication governance", + "Record and end consent, suppression, and channel-preference facts.", + ), + _permission( + "addresses:sync:read", + "View address sync", + "Inspect address sync sources, conflicts, tombstones, and diagnostics.", + ), + _permission( + "addresses:sync:write", + "Manage address sync", + "Bind address books to external sources and record sync state.", + ), + _permission( + "addresses:sync:admin", + "Administer address sync", + "Administer address sync connectors and future destructive sync operations.", + ), ) @@ -153,7 +234,13 @@ ROLE_TEMPLATES = ( slug="address_book_reader", name="Address book reader", description="Read visible address books and contacts.", - permissions=("addresses:address_book:read", "addresses:address_list:read", "addresses:contact:read", "addresses:governance:read", "addresses:sync:read"), + permissions=( + "addresses:address_book:read", + "addresses:address_list:read", + "addresses:contact:read", + "addresses:governance:read", + "addresses:sync:read", + ), ), ) @@ -172,15 +259,42 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]: ) return { - "address_books": session.query(AddressBook).filter(AddressBook.tenant_id == tenant_id, AddressBook.deleted_at.is_(None)).count(), - "address_lists": session.query(AddressList).filter(AddressList.tenant_id == tenant_id, AddressList.deleted_at.is_(None)).count(), - "contacts": session.query(Contact).filter(Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None)).count(), - "active_contact_merges": session.query(ContactMergeRecord).filter(ContactMergeRecord.tenant_id == tenant_id, ContactMergeRecord.status == "active").count(), - "contact_quality_decisions": session.query(ContactPointQualityDecision).filter(ContactPointQualityDecision.tenant_id == tenant_id).count(), - "contact_point_snapshots": session.query(ContactPointSnapshot).filter(ContactPointSnapshot.tenant_id == tenant_id).count(), - "sync_sources": session.query(AddressSyncSource).filter(AddressSyncSource.tenant_id == tenant_id, AddressSyncSource.enabled.is_(True)).count(), - "address_import_profiles": session.query(AddressImportProfile).filter(AddressImportProfile.tenant_id == tenant_id, AddressImportProfile.is_current.is_(True)).count(), - "address_import_runs": session.query(AddressImportRun).filter(AddressImportRun.tenant_id == tenant_id).count(), + "address_books": session.query(AddressBook) + .filter(AddressBook.tenant_id == tenant_id, AddressBook.deleted_at.is_(None)) + .count(), + "address_lists": session.query(AddressList) + .filter(AddressList.tenant_id == tenant_id, AddressList.deleted_at.is_(None)) + .count(), + "contacts": session.query(Contact) + .filter(Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None)) + .count(), + "active_contact_merges": session.query(ContactMergeRecord) + .filter( + ContactMergeRecord.tenant_id == tenant_id, + ContactMergeRecord.status == "active", + ) + .count(), + "contact_quality_decisions": session.query(ContactPointQualityDecision) + .filter(ContactPointQualityDecision.tenant_id == tenant_id) + .count(), + "contact_point_snapshots": session.query(ContactPointSnapshot) + .filter(ContactPointSnapshot.tenant_id == tenant_id) + .count(), + "sync_sources": session.query(AddressSyncSource) + .filter( + AddressSyncSource.tenant_id == tenant_id, + AddressSyncSource.enabled.is_(True), + ) + .count(), + "address_import_profiles": session.query(AddressImportProfile) + .filter( + AddressImportProfile.tenant_id == tenant_id, + AddressImportProfile.is_current.is_(True), + ) + .count(), + "address_import_runs": session.query(AddressImportRun) + .filter(AddressImportRun.tenant_id == tenant_id) + .count(), } @@ -200,13 +314,28 @@ CARDDAV_PROVIDER = ExternalProviderDeclaration( ProviderObjectDeclaration( object_type="address_book", field_groups=("identity", "display", "sync_state"), - authority_modes=("external_authoritative", "external_mirror", "governed_sync"), + authority_modes=( + "external_authoritative", + "external_mirror", + "governed_sync", + ), default_authority_mode="external_mirror", ), ProviderObjectDeclaration( object_type="contact", - field_groups=("identity", "name", "postal", "email", "phone", "source_metadata"), - authority_modes=("external_authoritative", "external_mirror", "governed_sync"), + field_groups=( + "identity", + "name", + "postal", + "email", + "phone", + "source_metadata", + ), + authority_modes=( + "external_authoritative", + "external_mirror", + "governed_sync", + ), default_authority_mode="governed_sync", ), ), @@ -252,7 +381,15 @@ LDAP_PROVIDER = ExternalProviderDeclaration( objects=( ProviderObjectDeclaration( object_type="contact", - field_groups=("identity", "name", "organization", "postal", "email", "phone", "source_metadata"), + field_groups=( + "identity", + "name", + "organization", + "postal", + "email", + "phone", + "source_metadata", + ), authority_modes=("external_authoritative", "external_mirror"), default_authority_mode="external_authoritative", ), @@ -281,7 +418,11 @@ LDAP_PROVIDER = ExternalProviderDeclaration( reconciliation="Only a complete paged search may infer an absent source object and create a local tombstone.", outage="Existing contacts remain available and visibly stale; an unavailable directory never causes deletes.", classifications=("personal", "confidential", "restricted"), - purposes=("directory projection", "recipient resolution", "identity-linked contact discovery"), + purposes=( + "directory projection", + "recipient resolution", + "identity-linked contact discovery", + ), retention="Address, audit, and records policies govern local projections and tombstone evidence.", secret_handling="Bind secrets remain in reusable credential envelopes; URLs, previews, and diagnostics contain no credentials.", ), @@ -294,26 +435,71 @@ manifest = ModuleManifest( id="addresses", name="Addresses", version="0.1.18", - required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR), - optional_dependencies=("campaigns", "mail", "forms", "reporting", "portal", "postbox", "connectors"), + required_capabilities=( + CAPABILITY_AUTH_PRINCIPAL_RESOLVER, + CAPABILITY_AUTH_PERMISSION_EVALUATOR, + ), + optional_dependencies=( + "campaigns", + "mail", + "forms", + "reporting", + "portal", + "postbox", + "connectors", + ), provides_interfaces=( ModuleInterfaceProvider(name=CAPABILITY_ADDRESSES_LOOKUP, version="0.1.8"), - ModuleInterfaceProvider(name=CAPABILITY_ADDRESSES_PEOPLE_SEARCH, version="0.1.0"), - ModuleInterfaceProvider(name=CAPABILITY_ADDRESSES_RECIPIENT_SOURCE, version="0.1.9"), - ModuleInterfaceProvider(name=CAPABILITY_ADDRESSES_CONTACT_POINT_RESOLUTION, version="1.0.0"), - ModuleInterfaceProvider(name=CAPABILITY_ADDRESSES_CONTACT_WRITER, version="0.1.8"), - ModuleInterfaceProvider(name=CAPABILITY_RECIPIENT_CHANNEL_FACTS, version="0.1.0"), + ModuleInterfaceProvider( + name=CAPABILITY_ADDRESSES_PEOPLE_SEARCH, version="0.1.0" + ), + ModuleInterfaceProvider( + name=CAPABILITY_ADDRESSES_RECIPIENT_SOURCE, version="0.1.9" + ), + ModuleInterfaceProvider( + name=CAPABILITY_ADDRESSES_CONTACT_POINT_RESOLUTION, version="1.0.0" + ), + ModuleInterfaceProvider( + name=CAPABILITY_ADDRESSES_CONTACT_WRITER, version="0.1.8" + ), + ModuleInterfaceProvider( + name=CAPABILITY_RECIPIENT_CHANNEL_FACTS, version="0.1.0" + ), + ModuleInterfaceProvider(name=ADDRESSES_DSAR_CAPABILITY, version="0.1.0"), ), permissions=PERMISSIONS, route_factory=_addresses_router, role_templates=ROLE_TEMPLATES, tenant_summary_providers=(_tenant_summary,), - nav_items=(NavItem(path="/address-book", label="Address Book", icon="book-user", required_any=("addresses:contact:read",), order=80),), + nav_items=( + NavItem( + path="/address-book", + label="Address Book", + icon="book-user", + required_any=("addresses:contact:read",), + order=80, + ), + ), frontend=FrontendModule( module_id="addresses", package_name="@govoplan/addresses-webui", - routes=(FrontendRoute(path="/address-book", component="AddressBookPage", required_any=("addresses:contact:read",), order=80),), - nav_items=(NavItem(path="/address-book", label="Address Book", icon="book-user", required_any=("addresses:contact:read",), order=80),), + routes=( + FrontendRoute( + path="/address-book", + component="AddressBookPage", + required_any=("addresses:contact:read",), + order=80, + ), + ), + nav_items=( + NavItem( + path="/address-book", + label="Address Book", + icon="book-user", + required_any=("addresses:contact:read",), + order=80, + ), + ), product_areas=( ProductAreaContribution( id="people-responsibility", @@ -321,17 +507,56 @@ manifest = ModuleManifest( label="i18n:govoplan-core.product_area.people_responsibility", icon="users", description="i18n:govoplan-core.product_area.people_responsibility_description", - surface_ids=("addresses.nav.address.book", "addresses.route.address.book"), + surface_ids=( + "addresses.nav.address.book", + "addresses.route.address.book", + ), order=70, ), ), view_surfaces=( - ViewSurface(id="addresses.page", module_id="addresses", kind="route", label="Address Book", order=80), - ViewSurface(id="addresses.sources", module_id="addresses", kind="section", label="Address sources", order=10), - ViewSurface(id="addresses.contacts", module_id="addresses", kind="section", label="Contacts", order=20), - ViewSurface(id="addresses.detail", module_id="addresses", kind="section", label="Contact detail", order=30), - ViewSurface(id="addresses.governance", module_id="addresses", kind="action", label="Communication governance", order=40), - ViewSurface(id="addresses.sync", module_id="addresses", kind="action", label="Address synchronization", order=50), + ViewSurface( + id="addresses.page", + module_id="addresses", + kind="route", + label="Address Book", + order=80, + ), + ViewSurface( + id="addresses.sources", + module_id="addresses", + kind="section", + label="Address sources", + order=10, + ), + ViewSurface( + id="addresses.contacts", + module_id="addresses", + kind="section", + label="Contacts", + order=20, + ), + ViewSurface( + id="addresses.detail", + module_id="addresses", + kind="section", + label="Contact detail", + order=30, + ), + ViewSurface( + id="addresses.governance", + module_id="addresses", + kind="action", + label="Communication governance", + order=40, + ), + ViewSurface( + id="addresses.sync", + module_id="addresses", + kind="action", + label="Address synchronization", + order=50, + ), ), ), migration_spec=MigrationSpec( @@ -343,8 +568,13 @@ manifest = ModuleManifest( retirement_notes="Destructive retirement drops address-owned database tables after the installer captures a database snapshot.", ), capability_factories={ - CAPABILITY_ADDRESSES_LOOKUP: lambda context: __import__("govoplan_addresses.backend.capabilities", fromlist=["lookup_capability"]).lookup_capability(context), - CAPABILITY_ADDRESSES_PEOPLE_SEARCH: lambda context: __import__("govoplan_addresses.backend.capabilities", fromlist=["people_search_capability"]).people_search_capability(context), + CAPABILITY_ADDRESSES_LOOKUP: lambda context: __import__( + "govoplan_addresses.backend.capabilities", fromlist=["lookup_capability"] + ).lookup_capability(context), + CAPABILITY_ADDRESSES_PEOPLE_SEARCH: lambda context: __import__( + "govoplan_addresses.backend.capabilities", + fromlist=["people_search_capability"], + ).people_search_capability(context), CAPABILITY_ADDRESSES_RECIPIENT_SOURCE: lambda context: __import__( "govoplan_addresses.backend.capabilities", fromlist=["recipient_source_capability"], @@ -361,6 +591,20 @@ manifest = ModuleManifest( "govoplan_addresses.backend.capabilities", fromlist=["contact_point_resolution_capability"], ).contact_point_resolution_capability(context), + ADDRESSES_DSAR_CAPABILITY: _addresses_dsar_provider, + }, + capability_documentation={ + ADDRESSES_DSAR_CAPABILITY: CapabilityDocumentation( + label="Addresses data-subject request provider", + summary=( + "Finds bounded contact, contact-point, address-list, governance, " + "provenance, synchronization, and operator-attribution data without " + "exporting raw source payloads, connector state, or opaque evidence." + ), + contract_version="0.1.0", + documentation_types=("admin",), + audience=("privacy_officer", "addresses_admin", "records_manager"), + ), }, uninstall_guard_providers=( persistent_table_uninstall_guard( @@ -387,6 +631,57 @@ manifest = ModuleManifest( ), ), documentation=( + DocumentationTopic( + id="addresses.privacy.data-subject-requests", + title="Review Addresses data in a data-subject request", + summary=( + "Collect tenant-scoped contact data while preserving shared address, " + "recipient, synchronization, and provenance evidence." + ), + body=( + "Addresses searches corroborated email and account selectors plus " + "namespaced contact and contact-point references. A matching contact " + "exports bounded identity, email, telephone, and postal values together " + "with its address-list use and minimized governance, quality, provenance, " + "merge, redirect, and synchronization evidence. Account matches add only " + "minimized operator attribution for governed configuration and evidence. " + "The provider excludes raw imported or synchronized source payloads, " + "connector tokens and revisions that could act as credentials, opaque " + "metadata, snapshot request and resolution payloads, import plans, merge " + "before/after payloads, unrelated contacts, and other tenants. Quality, " + "governance, provenance, merge, redirect, synchronization, import, " + "snapshot, and operator evidence is retained with an explicit reason. " + "Because reusable contacts can be shared, synchronized, merged, or " + "referenced by immutable recipient snapshots, the DSAR provider never " + "deletes them automatically. An authorized operator must review " + "dependencies and use the normal Addresses correction, archive, source, " + "merge, or governance workflow." + ), + layer="static", + documentation_types=("admin",), + audience=( + "privacy_officer", + "addresses_admin", + "records_manager", + "operator", + ), + related_modules=( + "access", + "audit", + "campaigns", + "dist_lists", + "records", + ), + order=29, + metadata={ + "seed": True, + "help_contexts": [ + "addresses.contacts", + "addresses.governance", + "addresses.action.archive", + ], + }, + ), DocumentationTopic( id="addresses.boundary", title="Reusable address ownership", @@ -399,7 +694,14 @@ manifest = ModuleManifest( layer="configured", documentation_types=("admin", "user"), audience=("tenant_admin", "operator", "module_admin"), - related_modules=("campaigns", "mail", "forms", "reporting", "portal", "postbox"), + related_modules=( + "campaigns", + "mail", + "forms", + "reporting", + "portal", + "postbox", + ), order=30, metadata={ "seed": True, @@ -482,7 +784,11 @@ manifest = ModuleManifest( order=34, metadata={ "seed": True, - "help_contexts": ["addresses.action.import", "addresses.contacts", "addresses.sources"], + "help_contexts": [ + "addresses.action.import", + "addresses.contacts", + "addresses.sources", + ], }, ), DocumentationTopic( @@ -537,7 +843,14 @@ manifest = ModuleManifest( layer="configured", documentation_types=("admin", "user"), audience=("tenant_admin", "operator", "module_admin", "power_user"), - related_modules=("dist_lists", "connectors", "datasources", "campaigns", "policy", "audit"), + related_modules=( + "dist_lists", + "connectors", + "datasources", + "campaigns", + "policy", + "audit", + ), order=36, metadata={ "seed": True, @@ -579,15 +892,27 @@ manifest = ModuleManifest( maturity="vertical_slice", documentation_ref="docs/ADDRESS_MODULE_ARCHITECTURE.md", test_ref="tests/test_addresses_service.py", - known_limits=("External address-book synchronization remains a bounded connector slice rather than a supported provider profile.",), + known_limits=( + "External address-book synchronization remains a bounded connector slice rather than a supported provider profile.", + ), supported_authority_modes=( "native_authoritative", "external_authoritative", "external_mirror", "governed_sync", ), - owned_concepts=("contact point", "address book", "contact consent", "recipient source"), - non_owned_concepts=("identity", "organization", "campaign recipient snapshot", "procedure party"), + owned_concepts=( + "contact point", + "address book", + "contact consent", + "recipient source", + ), + non_owned_concepts=( + "identity", + "organization", + "campaign recipient snapshot", + "procedure party", + ), target_tested_providers=(CARDDAV_PROVIDER_ID,), security_docs=("docs/ADDRESS_MODULE_ARCHITECTURE.md",), operations_docs=("README.md",), diff --git a/tests/test_dsar_provider.py b/tests/test_dsar_provider.py new file mode 100644 index 0000000..b25f593 --- /dev/null +++ b/tests/test_dsar_provider.py @@ -0,0 +1,588 @@ +from __future__ import annotations + +import unittest +from datetime import datetime, timezone + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_access.backend.db.models import Account, User +from govoplan_addresses.backend.db.models import ( + AddressBook, + AddressImportRun, + AddressList, + AddressListEntry, + AddressSyncConflict, + AddressSyncSource, + AddressSyncTombstone, + Contact, + ContactChannelRule, + ContactEmail, + ContactFieldProvenance, + ContactMergeRecord, + ContactPhone, + ContactPointQualityDecision, + ContactPostalAddress, + ContactRedirect, +) +from govoplan_addresses.backend.dsar_provider import ( + ADDRESSES_DSAR_CAPABILITY, + AddressesDsarProvider, +) +from govoplan_addresses.backend.manifest import manifest +from govoplan_core.core.dsar import ( + DsarErasureActionRef, + DsarProvider, + DsarSubjectRef, +) +from govoplan_core.db.base import Base +from govoplan_core.privacy.dsar_workflow import ( + create_data_subject_request, + search_data_subject_request, +) + + +class _Registry: + def __init__( + self, + provider: AddressesDsarProvider, + *, + addresses_active: bool = True, + ) -> None: + self.provider = provider + self.addresses_active = addresses_active + + def capability_names(self): + return (ADDRESSES_DSAR_CAPABILITY,) + + def capability_owner(self, name): + self._assert_capability(name) + return "addresses" + + def tenant_entitlement_resolver(self): + addresses_active = self.addresses_active + + class _Resolver: + @staticmethod + def resolve(session, tenant_id): + del session, tenant_id + return type( + "State", + (), + {"effective_modules": (("addresses",) if addresses_active else ())}, + )() + + return _Resolver() + + def require_tenant_capability(self, name, session, **kwargs): + del session, kwargs + self._assert_capability(name) + return self.provider + + def manifests(self): + return (type("Manifest", (), {"id": "addresses"})(),) + + @staticmethod + def _assert_capability(name: str) -> None: + if name != ADDRESSES_DSAR_CAPABILITY: + raise KeyError(name) + + +class AddressesDsarProviderTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite:///:memory:", future=True) + Base.metadata.create_all(bind=self.engine) + self.session = sessionmaker(bind=self.engine, future=True)() + now = datetime.now(timezone.utc) + + self.account = Account( + id="account-1", + email="subject@example.test", + normalized_email="subject@example.test", + display_name="Subject", + password_hash="password-secret-do-not-export", + ) + self.user = User( + id="membership-1", + tenant_id="tenant-1", + account_id=self.account.id, + email=self.account.email, + display_name="Subject", + ) + self.book = AddressBook( + id="book-1", + tenant_id="tenant-1", + scope_type="tenant", + scope_id="tenant-1", + name="Residents", + created_by_account_id=self.account.id, + metadata_={"secret": "book-metadata-do-not-export"}, + ) + self.contact = Contact( + id="contact-1", + tenant_id="tenant-1", + address_book_id=self.book.id, + display_name="Subject Person", + given_name="Subject", + family_name="Person", + organization="Example household", + note="A bounded subject note", + tags=["resident"], + source_kind="carddav", + source_ref="https://source.invalid/private/contact.vcf", + source_payload_kind="vcard", + source_payload_raw="raw-source-payload-do-not-export", + source_revision="revision-7", + provenance={"secret": "contact-provenance-do-not-export"}, + created_by_account_id=self.account.id, + metadata_={"secret": "contact-metadata-do-not-export"}, + ) + self.email = ContactEmail( + id="email-1", + contact_id=self.contact.id, + label="private", + email="Subject@Example.test", + original_email="Subject@Example.test", + normalized_email="subject@example.test", + provenance={"secret": "email-provenance-do-not-export"}, + is_primary=True, + ) + self.phone = ContactPhone( + id="phone-1", + contact_id=self.contact.id, + label="mobile", + phone="+49 30 123456", + original_phone="030 123456", + normalized_phone="+4930123456", + provenance={"secret": "phone-provenance-do-not-export"}, + is_primary=True, + ) + self.postal = ContactPostalAddress( + id="postal-1", + contact_id=self.contact.id, + label="home", + street="Example Street 1", + postal_code="10115", + locality="Berlin", + country="DE", + original_value={"secret": "postal-original-do-not-export"}, + normalized_value={"secret": "postal-normalized-do-not-export"}, + provenance={"secret": "postal-provenance-do-not-export"}, + is_primary=True, + ) + self.address_list = AddressList( + id="list-1", + tenant_id="tenant-1", + address_book_id=self.book.id, + name="District residents", + created_by_account_id="another-account", + ) + self.list_entry = AddressListEntry( + id="entry-1", + address_list_id=self.address_list.id, + contact_id=self.contact.id, + contact_email_id=self.email.id, + target_kind="email", + metadata_={"secret": "list-entry-metadata-do-not-export"}, + ) + self.channel_rule = ContactChannelRule( + id="rule-1", + tenant_id="tenant-1", + contact_id=self.contact.id, + channel="email", + purpose="resident-notice", + contact_point_id=self.email.id, + decision="allow", + legal_basis="public task", + evidence_ref="records://consent/evidence-1", + reason="Current resident preference", + effective_from=now, + created_by_account_id="another-account", + metadata_={"secret": "rule-metadata-do-not-export"}, + ) + self.quality = ContactPointQualityDecision( + id="quality-1", + tenant_id="tenant-1", + contact_id=self.contact.id, + channel="email", + contact_point_id=self.email.id, + state="valid", + reason_code="verified", + reason="Verified by operator", + evidence_ref="files://private/evidence", + effective_from=now, + created_by_account_id="another-account", + metadata_={"secret": "quality-metadata-do-not-export"}, + ) + self.provenance = ContactFieldProvenance( + id="provenance-1", + tenant_id="tenant-1", + contact_id=self.contact.id, + field_path="emails[0].email", + value={"secret": "field-value-do-not-export"}, + source_kind="carddav", + source_ref="https://source.invalid/private", + source_revision="revision-7", + precedence=10, + selected=True, + reason_code="source_authority", + explanation="Selected from the authoritative source", + visibility="operator", + created_by_account_id="another-account", + metadata_={"secret": "field-metadata-do-not-export"}, + ) + self.sync_source = AddressSyncSource( + id="source-1", + tenant_id="tenant-1", + address_book_id=self.book.id, + connector_type="carddav", + display_name="Residents CardDAV", + external_account_ref="private-account-ref-do-not-export", + external_address_book_ref="private-book-ref-do-not-export", + sync_token="sync-token-do-not-export", + etag="private-etag-do-not-export", + remote_revision="private-remote-revision-do-not-export", + last_diagnostic={"secret": "diagnostic-do-not-export"}, + created_by_account_id=self.account.id, + metadata_={"secret": "source-metadata-do-not-export"}, + ) + self.tombstone = AddressSyncTombstone( + id="tombstone-1", + tenant_id="tenant-1", + sync_source_id=self.sync_source.id, + address_book_id=self.book.id, + contact_id=self.contact.id, + remote_uid="private-uid-do-not-export", + resource_href="private-href-do-not-export", + synced_at=now, + metadata_={"secret": "tombstone-metadata-do-not-export"}, + ) + self.conflict = AddressSyncConflict( + id="conflict-1", + tenant_id="tenant-1", + sync_source_id=self.sync_source.id, + address_book_id=self.book.id, + contact_id=self.contact.id, + remote_uid="private-conflict-uid-do-not-export", + resource_href="private-conflict-href-do-not-export", + field_path="family_name", + local_value={"secret": "local-value-do-not-export"}, + remote_value={"secret": "remote-value-do-not-export"}, + status="resolved", + resolution="local", + resolved_at=now, + resolved_by_account_id=self.account.id, + metadata_={"secret": "conflict-metadata-do-not-export"}, + ) + self.import_run = AddressImportRun( + id="import-1", + tenant_id="tenant-1", + address_book_id=self.book.id, + source_filename="contacts.csv", + source_format="csv", + input_hash="a" * 64, + plan_hash="b" * 64, + status="applied", + row_count=1, + statistics={"secret": "statistics-do-not-export"}, + diagnostics=[{"secret": "import-diagnostic-do-not-export"}], + plan_data=[{"secret": "import-plan-do-not-export"}], + result_evidence={"secret": "import-result-do-not-export"}, + created_by_account_id=self.account.id, + applied_at=now, + ) + self.merge = ContactMergeRecord( + id="merge-1", + tenant_id="tenant-1", + address_book_id=self.book.id, + winner_contact_id=self.contact.id, + loser_contact_ids=["old-contact-1"], + status="active", + reason="Duplicate contact", + survivorship={"secret": "survivorship-do-not-export"}, + decisions=[{"secret": "merge-decisions-do-not-export"}], + before_payload={"secret": "merge-before-do-not-export"}, + after_payload={"secret": "merge-after-do-not-export"}, + before_hash="c" * 64, + after_hash="d" * 64, + created_by_account_id="another-account", + provenance={"secret": "merge-provenance-do-not-export"}, + ) + self.redirect = ContactRedirect( + id="redirect-1", + tenant_id="tenant-1", + source_contact_id="old-contact-1", + target_contact_id=self.contact.id, + merge_record_id=self.merge.id, + ) + self.unrelated = Contact( + id="contact-unrelated", + tenant_id="tenant-1", + address_book_id=self.book.id, + display_name="Unrelated Person", + note="unrelated-person-do-not-export", + ) + unrelated_email = ContactEmail( + id="email-unrelated", + contact_id=self.unrelated.id, + email="unrelated@example.test", + original_email="unrelated@example.test", + normalized_email="unrelated@example.test", + ) + tenant_two_book = AddressBook( + id="book-tenant-2", + tenant_id="tenant-2", + scope_type="tenant", + scope_id="tenant-2", + name="Other tenant", + ) + tenant_two_contact = Contact( + id="contact-tenant-2", + tenant_id="tenant-2", + address_book_id=tenant_two_book.id, + display_name="Other Tenant Subject", + note="other-tenant-do-not-export", + ) + tenant_two_email = ContactEmail( + id="email-tenant-2", + contact_id=tenant_two_contact.id, + email="subject@example.test", + original_email="subject@example.test", + normalized_email="subject@example.test", + ) + self.session.add_all( + [ + self.account, + self.user, + self.book, + self.contact, + self.email, + self.phone, + self.postal, + self.address_list, + self.list_entry, + self.channel_rule, + self.quality, + self.provenance, + self.sync_source, + self.tombstone, + self.conflict, + self.import_run, + self.merge, + self.redirect, + self.unrelated, + unrelated_email, + tenant_two_book, + tenant_two_contact, + tenant_two_email, + ] + ) + self.session.commit() + self.provider = AddressesDsarProvider() + self.subject = DsarSubjectRef( + account_id=self.account.id, + email=self.account.email, + ) + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_manifest_publishes_protocol_conforming_provider(self) -> None: + provided_names = {item.name for item in manifest.provides_interfaces} + self.assertIn(ADDRESSES_DSAR_CAPABILITY, provided_names) + provider = manifest.capability_factories[ADDRESSES_DSAR_CAPABILITY](None) + self.assertIsInstance(provider, DsarProvider) + + def test_search_is_tenant_scoped_related_and_minimized(self) -> None: + records = self.provider.search_subject( + self.session, + tenant_id="tenant-1", + subject=self.subject, + ) + + resource_types = {record.resource_type for record in records} + self.assertTrue( + { + "addresses_contact", + "addresses_contact_email", + "addresses_contact_phone", + "addresses_contact_postal_address", + "addresses_list_membership", + "addresses_channel_rule", + "addresses_quality_decision", + "addresses_field_provenance", + "addresses_merge_record", + "addresses_contact_redirect", + "addresses_sync_tombstone", + "addresses_sync_conflict", + "addresses_address_book_attribution", + "addresses_sync_source_attribution", + "addresses_import_run_attribution", + }.issubset(resource_types) + ) + serialized = repr([record.to_dict() for record in records]) + excluded_values = ( + "password-secret-do-not-export", + "raw-source-payload-do-not-export", + "contact-provenance-do-not-export", + "book-metadata-do-not-export", + "field-value-do-not-export", + "sync-token-do-not-export", + "private-account-ref-do-not-export", + "private-remote-revision-do-not-export", + "local-value-do-not-export", + "remote-value-do-not-export", + "import-plan-do-not-export", + "merge-before-do-not-export", + "merge-after-do-not-export", + "unrelated-person-do-not-export", + "other-tenant-do-not-export", + ) + for value in excluded_values: + self.assertNotIn(value, serialized) + + def test_conflicting_selectors_and_uncorroborated_reference_fail_closed( + self, + ) -> None: + conflict = self.provider.search_subject( + self.session, + tenant_id="tenant-1", + subject=DsarSubjectRef( + email="subject@example.test", + external_references={"addresses.email": "other@example.test"}, + ), + ) + uncorroborated = self.provider.search_subject( + self.session, + tenant_id="tenant-1", + subject=DsarSubjectRef( + email="subject@example.test", + external_references={"addresses.contact": self.unrelated.id}, + ), + ) + + self.assertEqual((), conflict) + self.assertEqual((), uncorroborated) + + def test_plan_retains_evidence_and_routes_contact_data_to_review(self) -> None: + records = self.provider.search_subject( + self.session, + tenant_id="tenant-1", + subject=self.subject, + ) + actions = self.provider.plan_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + records=records, + ) + + kinds = {action.kind for action in actions} + self.assertEqual({"manual_review", "retain"}, kinds) + self.assertFalse(any(action.executable for action in actions)) + retained = [action for action in actions if action.kind == "retain"] + self.assertTrue(retained) + self.assertTrue(all(action.rationale for action in retained)) + + results = self.provider.execute_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + actions=actions, + request_id="dsar-addresses-1", + ) + self.assertEqual({"blocked"}, {result.status for result in results}) + self.assertIsNotNone(self.session.get(Contact, self.contact.id)) + + def test_execution_rejects_foreign_or_forged_executable_actions(self) -> None: + foreign = DsarErasureActionRef( + action_id="mail:delete:contact:contact-1", + provider_id="mail", + module_id="mail", + kind="delete", + resource_type="addresses_contact", + resource_id=self.contact.id, + title="Foreign delete", + rationale="Must be rejected", + executable=True, + ) + forged = DsarErasureActionRef( + action_id="addresses:delete:addresses_contact:contact-1", + provider_id="addresses", + module_id="addresses", + kind="delete", + resource_type="addresses_contact", + resource_id=self.contact.id, + title="Forged delete", + rationale="Must be rejected", + executable=True, + ) + + with self.assertRaises(ValueError): + self.provider.execute_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + actions=(foreign,), + request_id="dsar-addresses-2", + ) + with self.assertRaises(ValueError): + self.provider.execute_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + actions=(forged,), + request_id="dsar-addresses-2", + ) + + def test_core_workflow_discovers_active_and_inactive_provider(self) -> None: + request = create_data_subject_request( + self.session, + tenant_id="tenant-1", + reference="DSAR-ADDRESSES-1", + request_kind="access", + subject=self.subject, + purpose="Respond to an authorized privacy request.", + legal_basis="Article 15 GDPR", + due_at=None, + requested_by_account_id="privacy-officer", + ) + self.session.commit() + search_data_subject_request( + self.session, + registry=_Registry(self.provider), + row=request, + expected_revision=1, + ) + self.assertEqual("searched", request.status) + self.assertEqual(["addresses"], request.coverage["covered_modules"]) + self.assertEqual([], request.coverage["modules_without_provider"]) + + disabled = create_data_subject_request( + self.session, + tenant_id="tenant-1", + reference="DSAR-ADDRESSES-DISABLED", + request_kind="access", + subject=self.subject, + purpose="Verify disabled-module coverage.", + legal_basis="Article 15 GDPR", + due_at=None, + requested_by_account_id="privacy-officer", + ) + search_data_subject_request( + self.session, + registry=_Registry(self.provider, addresses_active=False), + row=disabled, + expected_revision=1, + ) + + self.assertEqual(0, disabled.search_result["record_count"]) + self.assertEqual( + [ADDRESSES_DSAR_CAPABILITY], + disabled.coverage["inactive_provider_capabilities"], + ) + + +if __name__ == "__main__": + unittest.main()