From a2a9e8e814ce09b9329bcf97cd8fb18051a20e20 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 20 Aug 2026 23:27:27 +0200 Subject: [PATCH] feat: add governed Calendar DSAR coverage --- README.md | 9 + docs/CALENDAR_INTEGRATION_CONCEPT.md | 29 + .../backend/dsar_provider.py | 888 ++++++++++++++++++ src/govoplan_calendar/backend/manifest.py | 80 +- tests/test_dsar_provider.py | 597 ++++++++++++ 5 files changed, 1602 insertions(+), 1 deletion(-) create mode 100644 src/govoplan_calendar/backend/dsar_provider.py create mode 100644 tests/test_dsar_provider.py diff --git a/README.md b/README.md index 55bd11b..6f55d87 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,15 @@ and appointment modules. It is not yet a CalDAV network server: external clients cannot use GovOPlaN itself as their CalDAV endpoint, and scheduling inbox/outbox delivery remains owned by the scheduling and mail integration work. +Calendar also publishes `privacy.dsar.calendar`. Core's governed +data-subject-request workflow can use it to find tenant-scoped organizer, +attendee, preference, synchronization, outbox, and migration metadata. The +provider isolates the matching party and omits raw ICS, connector locators, +credentials, tokens, worker claims, and unrelated attendees. It classifies +synchronized and correlated state as retained evidence, leaves shared event +changes for manual review, and can safely delete only the subject's personal +view preference after revalidating tenant and ownership. + ## Development Install through the core environment: diff --git a/docs/CALENDAR_INTEGRATION_CONCEPT.md b/docs/CALENDAR_INTEGRATION_CONCEPT.md index 08f80a9..1fd11d8 100644 --- a/docs/CALENDAR_INTEGRATION_CONCEPT.md +++ b/docs/CALENDAR_INTEGRATION_CONCEPT.md @@ -191,6 +191,35 @@ the same tenant scope. ## Integration Points +## Data-subject requests and retention boundaries + +Calendar implements the optional `privacy.dsar.calendar` capability used by +Core's governed data-subject-request workflow. A search is bounded to the +effective tenant and matches normalized organizer or attendee email, direct +membership references, and independently corroborated namespaced Calendar +collection or event references. Only the matching party fragment is projected; +unrelated attendees and unrelated events in the same collection are not copied +into the case. + +The projection includes safe event and collection context, personal view +preferences, subject-owned synchronization configuration, outbox outcomes, and +migration state. It never includes raw ICS or complete iCalendar objects, +collection URLs, remote resource hrefs or ETags, sync tokens, usernames, +credential references or ciphertext, idempotency keys, worker leases, provider +error text, or opaque metadata. An authorized reviewer must use Calendar's own +screens when excluded content is necessary to decide the request. + +Synchronized, correlated, deleted, queued, and migrated state is classified as +retained evidence with an explicit reason. Local collections and events and +active credential metadata remain manual-review items because erasure can +affect recurrence, other attendees, institutional scheduling, remote systems, +and retention duties. The only executable provider action is deletion of the +subject's personal view preference; execution locks and revalidates its tenant +and owner and is idempotent. Event, attendee, credential, outbox, and migration +records are never mutated directly by the DSAR provider. Related meeting-poll, +mail, and delivery data remains owned by Scheduling, Poll, Mail, and their own +providers. + ### Scheduling `govoplan-scheduling` should use calendar for: diff --git a/src/govoplan_calendar/backend/dsar_provider.py b/src/govoplan_calendar/backend/dsar_provider.py new file mode 100644 index 0000000..c08c3f2 --- /dev/null +++ b/src/govoplan_calendar/backend/dsar_provider.py @@ -0,0 +1,888 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone + +from sqlalchemy import Text, cast, func, or_ +from sqlalchemy.orm import Session + +from govoplan_calendar.backend.db.models import ( + CalendarCollection, + CalendarEvent, + CalendarMigrationBatch, + CalendarMigrationResource, + CalendarOutboxOperation, + CalendarSyncCredential, + CalendarSyncSource, + CalendarViewPreference, +) +from govoplan_core.core.dsar import ( + DsarErasureActionRef, + DsarExecutionResultRef, + DsarRecordRef, + DsarSubjectRef, + dsar_capability_name, +) + + +CALENDAR_DSAR_CAPABILITY = dsar_capability_name("calendar") +_MAX_RECORDS = 5_000 +_SECRET_PARTS = ( + "authorization", + "credential", + "idempotency", + "lease", + "password", + "secret", + "token", +) +_LOCATOR_PARTS = ("href", "path", "url") + + +class CalendarDsarProvider: + provider_id = "calendar" + module_id = "calendar" + + def search_subject( + self, + session: object, + *, + tenant_id: str, + subject: DsarSubjectRef, + ) -> Sequence[DsarRecordRef]: + db = _session(session) + subject_user_id = _subject_user_id(subject) + subject_email = _subject_email(subject) + references = _calendar_references(subject) + if subject_user_id is None and subject_email is None and not references: + return () + + records: list[DsarRecordRef] = [] + + def append(record: DsarRecordRef) -> None: + if len(records) >= _MAX_RECORDS: + raise ValueError( + "Calendar DSAR match limit exceeded; narrow the subject selectors." + ) + records.append(record) + + events = _matching_events( + db, + tenant_id=tenant_id, + subject_user_id=subject_user_id, + subject_email=subject_email, + event_id=references.get("event"), + ) + event_ids = {row.id for row in events} + collection_ids = {row.calendar_id for row in events} + if references.get("collection"): + collection_ids.add(references["collection"]) + + collections = _matching_collections( + db, + tenant_id=tenant_id, + subject_user_id=subject_user_id, + collection_ids=collection_ids, + ) + collection_ids = {row.id for row in collections} + owned_collection_ids = { + row.id + for row in collections + if subject_user_id is not None + and ( + (row.owner_type == "user" and row.owner_id == subject_user_id) + or row.created_by_user_id == subject_user_id + ) + } + + outbox_by_event = _outbox_by_event( + db, + tenant_id=tenant_id, + event_ids=event_ids, + ) + for collection in collections: + match_fields = _matching_fields( + collection, + subject_user_id, + ("created_by_user_id",), + ) + if ( + subject_user_id is not None + and collection.owner_type == "user" + and collection.owner_id == subject_user_id + ): + match_fields.insert(0, "owner_id") + immutable = collection.deleted_at is not None + append( + _record( + "calendar_collection", + collection.id, + "calendar_collection", + collection.name, + { + "match_fields": match_fields, + "event_context": collection.id + in {row.calendar_id for row in events}, + "slug": collection.slug, + "name": collection.name, + "description": collection.description, + "timezone": collection.timezone, + "color": collection.color, + "owner_type": collection.owner_type, + "visibility": collection.visibility, + "is_default": collection.is_default, + "deleted_at": _iso(collection.deleted_at), + }, + observed_at=collection.updated_at, + immutable=immutable, + retention_reason=( + "Deleted collection state is retained until Calendar lifecycle cleanup completes." + if immutable + else None + ), + source_path="/calendar", + ) + ) + + for event in events: + organizer = _matching_party(event.organizer, subject_email) + attendees = _matching_parties(event.attendees, subject_email) + actor_fields = _matching_fields( + event, + subject_user_id, + ("created_by_user_id", "updated_by_user_id"), + ) + immutable = bool( + event.deleted_at + or event.source_kind != "local" + or event.correlation_key + or event.producer_module + or outbox_by_event.get(event.id) + ) + append( + _record( + "calendar_event", + event.id, + "calendar_event", + event.summary, + { + "match_fields": actor_fields + + (["organizer"] if organizer else []) + + (["attendees"] if attendees else []), + "calendar_id": event.calendar_id, + "uid": event.uid, + "recurrence_id": event.recurrence_id, + "sequence": event.sequence, + "summary": event.summary, + "description": event.description, + "location": event.location, + "status": event.status, + "transparency": event.transparency, + "classification": event.classification, + "start_at": _iso(event.start_at), + "end_at": _iso(event.end_at), + "duration_seconds": event.duration_seconds, + "all_day": event.all_day, + "timezone": event.timezone, + "organizer": organizer, + "matching_attendees": attendees, + "categories": _safe_value(event.categories), + "rrule": _safe_value(event.rrule), + "rdate": _safe_value(event.rdate), + "exdate": _safe_value(event.exdate), + "reminders": _safe_value(event.reminders), + "attachment_count": len(event.attachments or ()), + "source_kind": event.source_kind, + "producer_module": event.producer_module, + "producer_resource_type": event.producer_resource_type, + "producer_resource_id": event.producer_resource_id, + "deleted_at": _iso(event.deleted_at), + }, + observed_at=event.updated_at, + immutable=immutable, + retention_reason=( + "Synchronized, correlated, deleted, or externally queued event state is retained as Calendar execution evidence." + if immutable + else None + ), + source_path="/calendar", + ) + ) + + for operations in outbox_by_event.values(): + for operation in operations: + append( + _record( + "calendar_outbox_operation", + operation.id, + "calendar_sync_evidence", + f"Calendar {operation.operation_kind} operation", + { + "event_id": operation.event_id, + "operation_kind": operation.operation_kind, + "payload_fingerprint": operation.payload_fingerprint, + "status": operation.status, + "attempt_count": operation.attempt_count, + "max_attempts": operation.max_attempts, + "available_at": _iso(operation.available_at), + "last_attempt_at": _iso(operation.last_attempt_at), + "completed_at": _iso(operation.completed_at), + "reconciled_at": _iso(operation.reconciled_at), + }, + observed_at=operation.updated_at, + immutable=True, + retention_reason="Calendar outbox outcomes are synchronization and recovery evidence.", + ) + ) + + self._append_user_resources( + db, + append=append, + tenant_id=tenant_id, + subject_user_id=subject_user_id, + ) + self._append_sync_and_migration_resources( + db, + append=append, + tenant_id=tenant_id, + subject_user_id=subject_user_id, + owned_collection_ids=owned_collection_ids, + ) + return tuple(records) + + def plan_erasure( + self, + session: object, + *, + tenant_id: str, + subject: DsarSubjectRef, + records: Sequence[DsarRecordRef], + ) -> Sequence[DsarErasureActionRef]: + del session + subject_user_id = _subject_user_id(subject) + actions: list[DsarErasureActionRef] = [] + for record in records: + if ( + record.provider_id != self.provider_id + or record.module_id != self.module_id + ): + raise ValueError("Calendar DSAR received a foreign provider record.") + actions.append( + _action( + f"calendar:{'retain' if record.immutable_evidence else 'review'}:{record.resource_type}:{record.resource_id}", + "retain" if record.immutable_evidence else "manual_review", + record, + f"{'Retain' if record.immutable_evidence else 'Review'} {record.title}", + record.retention_reason + or "Calendar content can span recurrence, attendees, synchronized resources, and external recovery state; review it through Calendar lifecycle controls.", + executable=False, + ) + ) + if ( + record.resource_type == "calendar_view_preference" + and "user_id" in record.data.get("match_fields", ()) + and subject_user_id is not None + ): + actions.append( + _action( + f"calendar:delete:calendar_view_preference:{record.resource_id}", + "delete", + record, + "Delete personal Calendar view preference", + "The personal presentation preference can be removed without changing events or synchronization evidence.", + executable=True, + irreversible=True, + metadata={ + "subject_user_id": subject_user_id, + "tenant_id": tenant_id, + }, + ) + ) + action_ids = [action.action_id for action in actions] + if len(action_ids) != len(set(action_ids)): + raise ValueError("Calendar DSAR produced duplicate action ids.") + return tuple(actions) + + def execute_erasure( + self, + session: object, + *, + tenant_id: str, + subject: DsarSubjectRef, + actions: Sequence[DsarErasureActionRef], + request_id: str, + ) -> Sequence[DsarExecutionResultRef]: + db = _session(session) + subject_user_id = _subject_user_id(subject) + results: list[DsarExecutionResultRef] = [] + for action in actions: + if ( + subject_user_id is None + or action.provider_id != self.provider_id + or action.module_id != self.module_id + or action.metadata.get("subject_user_id") != subject_user_id + or action.metadata.get("tenant_id") != tenant_id + ): + results.append( + _blocked(action, "The Calendar DSAR action is stale or invalid.") + ) + continue + if action.action_id.startswith("calendar:delete:calendar_view_preference:"): + results.append( + _delete_view_preference( + db, + tenant_id=tenant_id, + subject_user_id=subject_user_id, + action=action, + request_id=request_id, + ) + ) + else: + results.append( + _blocked(action, "Calendar does not execute this action kind.") + ) + db.flush() + return tuple(results) + + def _append_user_resources( + self, + db: Session, + *, + append: object, + tenant_id: str, + subject_user_id: str | None, + ) -> None: + if subject_user_id is None: + return + for preference in _bounded_rows( + db.query(CalendarViewPreference) + .filter( + CalendarViewPreference.tenant_id == tenant_id, + CalendarViewPreference.user_id == subject_user_id, + ) + .order_by(CalendarViewPreference.id) + ): + append( # type: ignore[operator] + _record( + "calendar_view_preference", + preference.id, + "personal_calendar_preference", + "Calendar view preference", + { + "match_fields": ["user_id"], + "dim_weekends": preference.dim_weekends, + "dim_off_hours": preference.dim_off_hours, + "workday_start_hour": preference.workday_start_hour, + "workday_end_hour": preference.workday_end_hour, + "continuous_virtualization": preference.continuous_virtualization, + "continuous_overscan_weeks": preference.continuous_overscan_weeks, + "alternate_continuous_months": preference.alternate_continuous_months, + }, + observed_at=preference.updated_at, + source_path="/settings?section=calendar", + ) + ) + for credential in _bounded_rows( + db.query(CalendarSyncCredential) + .filter( + CalendarSyncCredential.tenant_id == tenant_id, + CalendarSyncCredential.created_by_user_id == subject_user_id, + ) + .order_by(CalendarSyncCredential.id) + ): + append( # type: ignore[operator] + _record( + "calendar_sync_credential", + credential.id, + "calendar_sync_configuration", + credential.label or "Calendar sync credential", + { + "match_fields": ["created_by_user_id"], + "credential_kind": credential.credential_kind, + "label": credential.label, + "deleted_at": _iso(credential.deleted_at), + }, + observed_at=credential.updated_at, + immutable=credential.deleted_at is not None, + retention_reason=( + "Retired credential metadata remains synchronization governance evidence." + if credential.deleted_at is not None + else None + ), + ) + ) + + def _append_sync_and_migration_resources( + self, + db: Session, + *, + append: object, + tenant_id: str, + subject_user_id: str | None, + owned_collection_ids: set[str], + ) -> None: + if owned_collection_ids: + for source in _bounded_rows( + db.query(CalendarSyncSource) + .filter( + CalendarSyncSource.tenant_id == tenant_id, + CalendarSyncSource.calendar_id.in_(owned_collection_ids), + ) + .order_by(CalendarSyncSource.id) + ): + append( # type: ignore[operator] + _record( + "calendar_sync_source", + source.id, + "calendar_sync_configuration", + source.display_name or source.source_kind, + { + "calendar_id": source.calendar_id, + "source_kind": source.source_kind, + "display_name": source.display_name, + "auth_type": source.auth_type, + "sync_enabled": source.sync_enabled, + "sync_interval_seconds": source.sync_interval_seconds, + "sync_direction": source.sync_direction, + "conflict_policy": source.conflict_policy, + "last_attempt_at": _iso(source.last_attempt_at), + "last_synced_at": _iso(source.last_synced_at), + "next_sync_at": _iso(source.next_sync_at), + "last_status": source.last_status, + "deleted_at": _iso(source.deleted_at), + }, + observed_at=source.updated_at, + immutable=True, + retention_reason="External source configuration and synchronization state require coordinated Calendar review.", + ) + ) + + migration_conditions = [] + if subject_user_id is not None: + migration_conditions.append( + CalendarMigrationBatch.created_by_user_id == subject_user_id + ) + if owned_collection_ids: + migration_conditions.extend( + ( + CalendarMigrationBatch.source_calendar_id.in_(owned_collection_ids), + CalendarMigrationBatch.target_calendar_id.in_(owned_collection_ids), + ) + ) + if not migration_conditions: + return + batches = _bounded_rows( + db.query(CalendarMigrationBatch) + .filter( + CalendarMigrationBatch.tenant_id == tenant_id, + or_(*migration_conditions), + ) + .order_by(CalendarMigrationBatch.id) + ) + batch_ids = {row.id for row in batches} + for batch in batches: + append( # type: ignore[operator] + _record( + "calendar_migration_batch", + batch.id, + "calendar_migration_evidence", + f"Calendar migration {batch.id}", + { + "match_fields": _matching_fields( + batch, subject_user_id, ("created_by_user_id",) + ), + "migration_kind": batch.migration_kind, + "status": batch.status, + "phase": batch.phase, + "total_resources": batch.total_resources, + "total_events": batch.total_events, + "completed_at": _iso(batch.completed_at), + }, + observed_at=batch.updated_at, + immutable=True, + retention_reason="Remote-move authorization and outcome state is immutable migration evidence.", + ) + ) + if not batch_ids: + return + for resource in _bounded_rows( + db.query(CalendarMigrationResource) + .filter(CalendarMigrationResource.batch_id.in_(batch_ids)) + .order_by(CalendarMigrationResource.id) + ): + append( # type: ignore[operator] + _record( + "calendar_migration_resource", + resource.id, + "calendar_migration_evidence", + "Calendar migration resource", + { + "batch_id": resource.batch_id, + "status": resource.status, + "event_count": len(resource.event_ids or ()), + }, + observed_at=resource.updated_at, + immutable=True, + retention_reason="Per-resource move state is immutable migration and recovery evidence.", + ) + ) + + +def _matching_events( + session: Session, + *, + tenant_id: str, + subject_user_id: str | None, + subject_email: str | None, + event_id: str | None, +) -> list[CalendarEvent]: + conditions = [] + if event_id: + conditions.append(CalendarEvent.id == event_id) + if subject_user_id: + conditions.extend( + ( + CalendarEvent.created_by_user_id == subject_user_id, + CalendarEvent.updated_by_user_id == subject_user_id, + ) + ) + if subject_email: + pattern = f"%{_escape_like(subject_email)}%" + conditions.extend( + ( + func.lower(cast(CalendarEvent.organizer, Text)).like( + pattern, escape="\\" + ), + func.lower(cast(CalendarEvent.attendees, Text)).like( + pattern, escape="\\" + ), + ) + ) + if not conditions: + return [] + candidates = _bounded_rows( + session.query(CalendarEvent) + .filter(CalendarEvent.tenant_id == tenant_id, or_(*conditions)) + .order_by(CalendarEvent.id) + ) + return [ + row + for row in candidates + if row.id == event_id + or ( + subject_user_id is not None + and ( + row.created_by_user_id == subject_user_id + or row.updated_by_user_id == subject_user_id + ) + ) + or _matching_party(row.organizer, subject_email) + or _matching_parties(row.attendees, subject_email) + ] + + +def _matching_collections( + session: Session, + *, + tenant_id: str, + subject_user_id: str | None, + collection_ids: set[str], +) -> list[CalendarCollection]: + conditions = [] + if collection_ids: + conditions.append(CalendarCollection.id.in_(collection_ids)) + if subject_user_id: + conditions.extend( + ( + (CalendarCollection.owner_type == "user") + & (CalendarCollection.owner_id == subject_user_id), + CalendarCollection.created_by_user_id == subject_user_id, + ) + ) + if not conditions: + return [] + return _bounded_rows( + session.query(CalendarCollection) + .filter(CalendarCollection.tenant_id == tenant_id, or_(*conditions)) + .order_by(CalendarCollection.id) + ) + + +def _outbox_by_event( + session: Session, + *, + tenant_id: str, + event_ids: set[str], +) -> dict[str, list[CalendarOutboxOperation]]: + if not event_ids: + return {} + result: dict[str, list[CalendarOutboxOperation]] = {} + for row in _bounded_rows( + session.query(CalendarOutboxOperation) + .filter( + CalendarOutboxOperation.tenant_id == tenant_id, + CalendarOutboxOperation.event_id.in_(event_ids), + ) + .order_by(CalendarOutboxOperation.id) + ): + result.setdefault(str(row.event_id), []).append(row) + return result + + +def _matching_party(value: object, email: str | None) -> dict[str, object] | None: + if email is None or not isinstance(value, Mapping): + return None + address = _party_email(value) + if address != email: + return None + params = value.get("params") + safe_params: dict[str, object] = {} + if isinstance(params, Mapping): + for key in ("CN", "CUTYPE", "PARTSTAT", "ROLE", "RSVP"): + if key in params: + safe_params[key] = _safe_value(params[key]) + return { + "email": address, + "name": value.get("name") or _first_value(safe_params.get("CN")), + "participation_status": _first_value(safe_params.get("PARTSTAT")), + "role": _first_value(safe_params.get("ROLE")), + "rsvp": _first_value(safe_params.get("RSVP")), + } + + +def _matching_parties( + values: object, + email: str | None, +) -> list[dict[str, object]]: + if not isinstance(values, list): + return [] + return [party for value in values[:512] if (party := _matching_party(value, email))] + + +def _party_email(value: Mapping[object, object]) -> str | None: + candidate = value.get("email") or value.get("address") or value.get("value") + if not isinstance(candidate, str): + return None + if candidate.casefold().startswith("mailto:"): + candidate = candidate[7:] + return _normalized_email(candidate) + + +def _safe_value(value: object, *, depth: int = 0) -> object: + if depth >= 5: + return "[depth-limited]" + if isinstance(value, Mapping): + result: dict[str, object] = {} + for key, item in list(value.items())[:128]: + normalized = str(key).casefold() + if any(part in normalized for part in _SECRET_PARTS + _LOCATOR_PARTS): + continue + result[str(key)] = _safe_value(item, depth=depth + 1) + return result + if isinstance(value, list): + return [_safe_value(item, depth=depth + 1) for item in value[:256]] + if isinstance(value, str): + return value[:2_000] + if value is None or isinstance(value, (bool, int, float)): + return value + return str(value)[:2_000] + + +def _delete_view_preference( + session: Session, + *, + tenant_id: str, + subject_user_id: str, + action: DsarErasureActionRef, + request_id: str, +) -> DsarExecutionResultRef: + row = ( + session.query(CalendarViewPreference) + .filter( + CalendarViewPreference.id == action.resource_id, + CalendarViewPreference.tenant_id == tenant_id, + ) + .with_for_update() + .one_or_none() + ) + if row is None: + return _result( + action, + "unchanged", + "The Calendar view preference was already absent.", + {"request_id": request_id}, + ) + if row.user_id != subject_user_id: + return _blocked(action, "The Calendar preference owner changed after planning.") + session.delete(row) + return _result( + action, + "executed", + "The personal Calendar view preference was deleted.", + {"request_id": request_id}, + ) + + +def _calendar_references(subject: DsarSubjectRef) -> dict[str, str]: + aliases = { + "calendar.collection": "collection", + "calendar.event": "event", + } + return { + target: value + for key, target in aliases.items() + if (value := str(subject.external_references.get(key) or "").strip()) + } + + +def _subject_user_id(subject: DsarSubjectRef) -> str | None: + candidates = [subject.membership_id] if subject.membership_id else [] + for key in ( + "calendar.user", + "calendar.membership", + "access.membership", + "membership_id", + ): + value = str(subject.external_references.get(key) or "").strip() + if value: + candidates.append(value) + normalized = {value.strip() for value in candidates if value.strip()} + return normalized.pop() if len(normalized) == 1 else None + + +def _subject_email(subject: DsarSubjectRef) -> str | None: + candidates = [subject.email] if subject.email else [] + for key in ("calendar.email", "calendar.attendee_email"): + value = str(subject.external_references.get(key) or "").strip() + if value: + candidates.append(value) + normalized = { + email for value in candidates if (email := _normalized_email(value)) is not None + } + return normalized.pop() if len(normalized) == 1 else None + + +def _normalized_email(value: object) -> str | None: + if not isinstance(value, str): + return None + normalized = value.strip().casefold() + if normalized.startswith("mailto:"): + normalized = normalized[7:] + return normalized or None + + +def _first_value(value: object) -> object: + if isinstance(value, list): + return value[0] if value else None + return value + + +def _matching_fields( + row: object, + subject_user_id: str | None, + fields: Sequence[str], +) -> list[str]: + if subject_user_id is None: + return [] + return [field for field in fields if getattr(row, field) == subject_user_id] + + +def _record( + resource_type: str, + resource_id: str, + category: str, + title: str, + data: Mapping[str, object], + *, + observed_at: datetime | None = None, + immutable: bool = False, + retention_reason: str | None = None, + source_path: str | None = None, +) -> DsarRecordRef: + return DsarRecordRef( + provider_id="calendar", + module_id="calendar", + resource_type=resource_type, + resource_id=resource_id, + category=category, + title=title, + data=data, + observed_at=observed_at, + immutable_evidence=immutable, + retention_reason=retention_reason, + source_path=source_path, + ) + + +def _action( + action_id: str, + kind: str, + record: DsarRecordRef, + title: str, + rationale: str, + *, + executable: bool, + irreversible: bool = False, + metadata: Mapping[str, object] | None = None, +) -> DsarErasureActionRef: + return DsarErasureActionRef( + action_id=action_id, + provider_id="calendar", + module_id="calendar", + kind=kind, # type: ignore[arg-type] + resource_type=record.resource_type, + resource_id=record.resource_id, + title=title, + rationale=rationale, + executable=executable, + irreversible=irreversible, + metadata=metadata or {}, + ) + + +def _result( + action: DsarErasureActionRef, + status: str, + summary: str, + evidence: Mapping[str, object] | None = None, +) -> DsarExecutionResultRef: + return DsarExecutionResultRef( + action_id=action.action_id, + status=status, # type: ignore[arg-type] + summary=summary, + evidence=evidence or {}, + ) + + +def _blocked(action: DsarErasureActionRef, summary: str) -> DsarExecutionResultRef: + return _result(action, "blocked", summary) + + +def _session(value: object) -> Session: + if not isinstance(value, Session): + raise TypeError("Calendar DSAR provider requires a SQLAlchemy session.") + return value + + +def _bounded_rows(query: object) -> list[object]: + rows = query.limit(_MAX_RECORDS + 1).all() # type: ignore[attr-defined] + if len(rows) > _MAX_RECORDS: + raise ValueError( + "Calendar DSAR match limit exceeded; narrow the subject selectors." + ) + return rows + + +def _escape_like(value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def _iso(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.isoformat() + + +__all__ = ["CALENDAR_DSAR_CAPABILITY", "CalendarDsarProvider"] diff --git a/src/govoplan_calendar/backend/manifest.py b/src/govoplan_calendar/backend/manifest.py index 59e0038..2c6c965 100644 --- a/src/govoplan_calendar/backend/manifest.py +++ b/src/govoplan_calendar/backend/manifest.py @@ -17,7 +17,9 @@ from govoplan_core.core.calendar import ( ) from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard from govoplan_core.core.modules import ( + CapabilityDocumentation, DocumentationCondition, + DocumentationLink, DocumentationTopic, FrontendModule, FrontendRoute, @@ -44,6 +46,7 @@ from govoplan_core.core.provider_governance import ( ) from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base +from govoplan_calendar.backend.dsar_provider import CALENDAR_DSAR_CAPABILITY from govoplan_calendar.backend.search_source import create_calendar_search_source @@ -466,6 +469,13 @@ def _calendar_external_profile_provider(context: ModuleContext) -> object: return SqlCalendarExternalProfileProvider() +def _calendar_dsar_provider(context: ModuleContext) -> object: + del context + from govoplan_calendar.backend.dsar_provider import CalendarDsarProvider + + return CalendarDsarProvider() + + def _caldav_provider_states(context): from govoplan_calendar.backend.provider_state import caldav_provider_states @@ -499,7 +509,7 @@ def _open_xchange_provider_states(context): manifest = ModuleManifest( id="calendar", name="Calendar", - version="0.1.18", + version="0.1.19", required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR), optional_dependencies=("mail", "tasks", "scheduling", "appointments", "workflow_engine", "notifications", "dms", "connectors", "search"), provides_interfaces=( @@ -507,6 +517,7 @@ manifest = ModuleManifest( ModuleInterfaceProvider(name="calendar.scheduling", version="0.1.9"), ModuleInterfaceProvider(name="calendar.invitations", version="0.2.0"), ModuleInterfaceProvider(name="calendar.external_profiles", version="0.1.0"), + ModuleInterfaceProvider(name=CALENDAR_DSAR_CAPABILITY, version="0.1.0"), ), requires_interfaces=( ModuleInterfaceRequirement( @@ -556,6 +567,63 @@ manifest = ModuleManifest( ), ), documentation=( + DocumentationTopic( + id="calendar.privacy.data-subject-requests", + title="Review Calendar data in a data-subject request", + summary="Collect tenant-scoped event, participation, preference, synchronization, and migration metadata without disclosing connector secrets.", + body=( + "Calendar's DSAR provider searches the effective tenant by normalized organizer or attendee email, direct membership references, and namespaced Calendar collection or event references. " + "It returns only the matching organizer or attendee fragment alongside authorized event and collection context, personal view preferences, owned synchronization configuration, durable outbox outcomes, and migration evidence. It excludes raw ICS, complete iCalendar payloads, collection URLs, remote hrefs and ETags, sync tokens, credentials, encrypted secrets, worker leases, idempotency material, provider error text, and unrelated participants or events. " + "Synchronized, correlated, deleted, queued, and migrated state remains retained with a reason because it is institutional delivery, reconciliation, or recovery evidence. Local event, collection, and active credential metadata requires coordinated manual review. The provider can idempotently delete the subject's personal Calendar view preference after revalidating tenant and ownership; it never mutates an event, attendee list, synchronized resource, credential, or migration record directly." + ), + layer="configured", + documentation_types=("admin",), + audience=("privacy_officer", "calendar_manager", "records_manager", "operator"), + order=17, + conditions=( + DocumentationCondition( + required_modules=("calendar", "access"), + any_scopes=( + "access:privacy:read", + "access:privacy:manage", + "access:privacy:erase", + ), + ), + ), + links=( + DocumentationLink( + label="Data-subject requests", + href="/admin?section=tenant-data-subject-requests", + kind="runtime", + ), + DocumentationLink( + label="Calendar integration concept", + href="govoplan-calendar/docs/CALENDAR_INTEGRATION_CONCEPT.md", + kind="repository", + ), + ), + related_modules=("access", "audit", "ops", "scheduling"), + metadata={ + "kind": "workflow", + "route": "/admin?section=tenant-data-subject-requests", + "screen": "Data-subject requests", + "help_contexts": ["admin.privacy.data-subject-requests"], + "prerequisites": [ + "The privacy request and Calendar selectors have been independently authorized and corroborated.", + "The reviewer understands the effective Calendar retention and synchronization-evidence obligations.", + ], + "steps": [ + "Run the Calendar provider search and review the isolated event, participation, preference, sync, outbox, and migration records.", + "Keep every evidence retention reason with the case decision and coordinate any content change through Calendar lifecycle controls.", + "Execute only an approved personal view-preference deletion.", + "Review related Scheduling or Mail records through their owning DSAR providers when those modules are installed.", + ], + "limitations": [ + "Raw ICS and connector locators are not embedded in the JSON export; authorized Calendar review remains authoritative.", + "Event and attendee erasure is manual because recurrence, shared participation, external synchronization, and institutional retention can overlap.", + ], + }, + ), DocumentationTopic( id="calendar.quick-access-and-product-area", title="Calendar in product navigation and Quick Access", @@ -708,6 +776,16 @@ manifest = ModuleManifest( CAPABILITY_CALENDAR_SCHEDULING: _calendar_scheduling_provider, CAPABILITY_CALENDAR_INVITATIONS: _calendar_invitation_provider, CAPABILITY_CALENDAR_EXTERNAL_PROFILES: _calendar_external_profile_provider, + CALENDAR_DSAR_CAPABILITY: _calendar_dsar_provider, + }, + capability_documentation={ + CALENDAR_DSAR_CAPABILITY: CapabilityDocumentation( + label="Calendar data-subject request provider", + summary="Finds isolated Calendar participation and lifecycle metadata and classifies governed erasure actions.", + contract_version="0.1.0", + documentation_types=("admin",), + audience=("privacy_officer", "calendar_manager", "records_manager"), + ), }, nav_items=(NavItem(path="/calendar", label="Calendar", icon="calendar", required_any=("calendar:event:read",), order=55),), frontend=FrontendModule( diff --git a/tests/test_dsar_provider.py b/tests/test_dsar_provider.py new file mode 100644 index 0000000..fee4b97 --- /dev/null +++ b/tests/test_dsar_provider.py @@ -0,0 +1,597 @@ +from __future__ import annotations + +import unittest +from datetime import datetime, timezone + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_access.backend.db.models import Account, Group, User +from govoplan_calendar.backend.db.models import ( + CalendarCollection, + CalendarEvent, + CalendarMigrationBatch, + CalendarMigrationResource, + CalendarOutboxOperation, + CalendarSyncCredential, + CalendarSyncSource, + CalendarViewPreference, +) +from govoplan_calendar.backend.dsar_provider import ( + CALENDAR_DSAR_CAPABILITY, + CalendarDsarProvider, +) +from govoplan_calendar.backend.manifest import manifest +from govoplan_core.core.change_sequence import ChangeSequenceEntry +from govoplan_core.core.dsar import DsarProvider, DsarSubjectRef +from govoplan_core.db.base import Base +from govoplan_core.privacy.dsar_workflow import ( + DataSubjectRequest, + create_data_subject_request, + execute_data_subject_erasure, + plan_data_subject_erasure, + search_data_subject_request, +) + + +class _Registry: + def __init__( + self, + provider: CalendarDsarProvider, + *, + calendar_active: bool = True, + ) -> None: + self.provider = provider + self.calendar_active = calendar_active + + def capability_names(self): + return (CALENDAR_DSAR_CAPABILITY,) + + def capability_owner(self, name): + self._assert_capability(name) + return "calendar" + + def tenant_entitlement_resolver(self): + calendar_active = self.calendar_active + + class _Resolver: + @staticmethod + def resolve(session, tenant_id): + del session, tenant_id + return type( + "State", + (), + {"effective_modules": ("calendar",) if calendar_active else ()}, + )() + + return _Resolver() + + def require_tenant_capability(self, name, session, **kwargs): + del session, kwargs + self._assert_capability(name) + return self.provider + + def manifests(self): + return (type("Manifest", (), {"id": "calendar"})(),) + + @staticmethod + def _assert_capability(name: str) -> None: + if name != CALENDAR_DSAR_CAPABILITY: + raise KeyError(name) + + +class CalendarDsarProviderTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite:///:memory:", future=True) + Base.metadata.create_all( + bind=self.engine, + tables=[ + Account.__table__, + User.__table__, + Group.__table__, + ChangeSequenceEntry.__table__, + DataSubjectRequest.__table__, + CalendarCollection.__table__, + CalendarEvent.__table__, + CalendarViewPreference.__table__, + CalendarSyncSource.__table__, + CalendarSyncCredential.__table__, + CalendarOutboxOperation.__table__, + CalendarMigrationBatch.__table__, + CalendarMigrationResource.__table__, + ], + ) + self.session = sessionmaker(bind=self.engine, future=True)() + now = datetime.now(timezone.utc) + account = Account( + id="account-1", + email="subject@example.test", + normalized_email="subject@example.test", + display_name="Subject", + ) + other_account = Account( + id="account-2", + email="other@example.test", + normalized_email="other@example.test", + display_name="Other", + ) + self.user = User( + id="membership-1", + tenant_id="tenant-1", + account_id=account.id, + email="subject@example.test", + display_name="Subject", + ) + other_user = User( + id="membership-2", + tenant_id="tenant-1", + account_id=other_account.id, + email="other@example.test", + display_name="Other", + ) + self.collection = CalendarCollection( + id="calendar-subject", + tenant_id="tenant-1", + slug="subject-calendar", + name="Subject calendar", + description="Subject-owned calendar", + owner_type="user", + owner_id=self.user.id, + visibility="private", + created_by_user_id=self.user.id, + metadata_={"secret": "collection-secret-do-not-export"}, + ) + target_collection = CalendarCollection( + id="calendar-target", + tenant_id="tenant-1", + slug="target-calendar", + name="Target calendar", + owner_type="user", + owner_id=self.user.id, + visibility="private", + created_by_user_id=self.user.id, + ) + other_collection = CalendarCollection( + id="calendar-other", + tenant_id="tenant-1", + slug="other-calendar", + name="Other calendar", + owner_type="user", + owner_id=other_user.id, + visibility="private", + created_by_user_id=other_user.id, + ) + tenant_two_collection = CalendarCollection( + id="calendar-tenant-2", + tenant_id="tenant-2", + slug="tenant-two", + name="Tenant two secret calendar", + owner_type="tenant", + visibility="tenant", + ) + self.event = CalendarEvent( + id="event-subject", + tenant_id="tenant-1", + calendar_id=self.collection.id, + uid="subject-event@example.test", + summary="Subject appointment", + description="Information visible to the subject", + location="Town hall", + start_at=now, + end_at=now, + organizer={ + "value": "mailto:organizer@example.test", + "params": {"CN": ["Organizer"]}, + }, + attendees=[ + { + "value": "mailto:Subject@Example.Test", + "params": {"CN": ["Subject"], "PARTSTAT": ["ACCEPTED"]}, + }, + { + "value": "mailto:other@example.test", + "params": {"CN": ["Unrelated Person"]}, + }, + ], + categories=["citizen-service"], + reminders=[{"minutes": 15, "token": "reminder-secret-do-not-export"}], + attachments=[{"href": "/private/attachment-do-not-export"}], + source_kind="caldav", + source_href="/remote/event-do-not-export.ics", + etag="event-etag-do-not-export", + raw_ics="raw-ics-do-not-export", + icalendar={"private": "icalendar-object-do-not-export"}, + metadata_={"secret": "event-secret-do-not-export"}, + ) + unrelated_event = CalendarEvent( + id="event-other", + tenant_id="tenant-1", + calendar_id=self.collection.id, + uid="unrelated@example.test", + summary="Unrelated event do not export", + description="Unrelated event body do not export", + start_at=now, + end_at=now, + organizer={"value": "mailto:other@example.test"}, + attendees=[], + ) + tenant_two_event = CalendarEvent( + id="event-tenant-2", + tenant_id="tenant-2", + calendar_id=tenant_two_collection.id, + uid="tenant-two@example.test", + summary="Tenant two event do not export", + start_at=now, + end_at=now, + organizer={"value": "mailto:subject@example.test"}, + attendees=[], + ) + self.source = CalendarSyncSource( + id="source-subject", + tenant_id="tenant-1", + calendar_id=self.collection.id, + source_kind="caldav", + collection_url="https://private.example.test/calendar-do-not-export", + display_name="Subject CalDAV", + auth_type="basic", + username="connector-user-do-not-export", + credential_ref="credential-ref-do-not-export", + sync_token="sync-token-do-not-export", + ctag="ctag-do-not-export", + last_status="failed", + last_error="provider-error-do-not-export", + metadata_={"secret": "source-secret-do-not-export"}, + ) + target_source = CalendarSyncSource( + id="source-target", + tenant_id="tenant-1", + calendar_id=target_collection.id, + source_kind="caldav", + collection_url="https://private.example.test/target-do-not-export", + display_name="Target CalDAV", + auth_type="none", + ) + credential = CalendarSyncCredential( + id="credential-subject", + tenant_id="tenant-1", + credential_kind="basic", + label="Subject credential", + secret_encrypted="ciphertext-do-not-export", + created_by_user_id=self.user.id, + metadata_={"password": "credential-password-do-not-export"}, + ) + self.preference = CalendarViewPreference( + id="preference-subject", + tenant_id="tenant-1", + user_id=self.user.id, + dim_weekends=True, + workday_start_hour=8, + workday_end_hour=17, + ) + operation = CalendarOutboxOperation( + id="operation-subject", + tenant_id="tenant-1", + source_id=self.source.id, + event_id=self.event.id, + operation_kind="put", + resource_href="/private/outbox-href-do-not-export", + payload_ics="outbox-payload-do-not-export", + payload_fingerprint="a" * 64, + expected_etag="expected-etag-do-not-export", + idempotency_key="idempotency-do-not-export", + status="succeeded", + available_at=now, + lease_token="lease-do-not-export", + remote_etag="remote-etag-do-not-export", + last_error="outbox-error-do-not-export", + metadata_={"secret": "outbox-secret-do-not-export"}, + ) + migration = CalendarMigrationBatch( + id="migration-subject", + tenant_id="tenant-1", + status="active", + phase="copying_destination", + source_calendar_id=self.collection.id, + target_calendar_id=target_collection.id, + source_sync_source_id=self.source.id, + target_sync_source_id=target_source.id, + total_resources=1, + total_events=1, + created_by_user_id=self.user.id, + authorization_evidence={"secret": "authorization-do-not-export"}, + last_error="migration-error-do-not-export", + ) + migration_resource = CalendarMigrationResource( + id="migration-resource-subject", + tenant_id="tenant-1", + batch_id=migration.id, + source_href="/source/private-do-not-export", + source_expected_etag="source-etag-do-not-export", + destination_href="/destination/private-do-not-export", + event_ids=[self.event.id], + status="copy_pending", + last_error="resource-error-do-not-export", + ) + self.session.add_all( + [ + account, + other_account, + self.user, + other_user, + self.collection, + target_collection, + other_collection, + tenant_two_collection, + self.event, + unrelated_event, + tenant_two_event, + self.source, + target_source, + credential, + self.preference, + operation, + migration, + migration_resource, + ] + ) + self.session.commit() + self.provider = CalendarDsarProvider() + self.subject = DsarSubjectRef( + membership_id=self.user.id, + email="subject@example.test", + ) + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_manifest_publishes_protocol_conforming_provider(self) -> None: + provided_names = {item.name for item in manifest.provides_interfaces} + self.assertIn(CALENDAR_DSAR_CAPABILITY, provided_names) + provider = manifest.capability_factories[CALENDAR_DSAR_CAPABILITY](None) + self.assertIsInstance(provider, DsarProvider) + self.assertIn( + "calendar.privacy.data-subject-requests", + {topic.id for topic in manifest.documentation}, + ) + + def test_search_is_tenant_scoped_minimized_and_party_specific(self) -> None: + records = self._records() + resource_types = {record.resource_type for record in records} + self.assertTrue( + { + "calendar_collection", + "calendar_event", + "calendar_view_preference", + "calendar_sync_credential", + "calendar_sync_source", + "calendar_outbox_operation", + "calendar_migration_batch", + "calendar_migration_resource", + }.issubset(resource_types) + ) + event = next( + record for record in records if record.resource_type == "calendar_event" + ) + self.assertEqual( + "subject@example.test", + event.data["matching_attendees"][0]["email"], + ) + self.assertEqual([], event.data["organizer"] or []) + + serialized = repr([record.to_dict() for record in records]) + for hidden in ( + "event-other", + "Unrelated event do not export", + "Unrelated event body do not export", + "other@example.test", + "Unrelated Person", + "event-tenant-2", + "Tenant two event do not export", + "raw-ics-do-not-export", + "icalendar-object-do-not-export", + "/remote/event-do-not-export.ics", + "event-etag-do-not-export", + "/private/attachment-do-not-export", + "collection-secret-do-not-export", + "event-secret-do-not-export", + "reminder-secret-do-not-export", + "calendar-do-not-export", + "connector-user-do-not-export", + "credential-ref-do-not-export", + "sync-token-do-not-export", + "ctag-do-not-export", + "provider-error-do-not-export", + "source-secret-do-not-export", + "ciphertext-do-not-export", + "credential-password-do-not-export", + "outbox-href-do-not-export", + "outbox-payload-do-not-export", + "expected-etag-do-not-export", + "idempotency-do-not-export", + "lease-do-not-export", + "remote-etag-do-not-export", + "outbox-error-do-not-export", + "authorization-do-not-export", + "migration-error-do-not-export", + "/source/private-do-not-export", + "source-etag-do-not-export", + "/destination/private-do-not-export", + "resource-error-do-not-export", + ): + self.assertNotIn(hidden, serialized) + + def test_conflicting_email_references_fail_closed_for_attendee_data(self) -> None: + records = self.provider.search_subject( + self.session, + tenant_id="tenant-1", + subject=DsarSubjectRef( + email="subject@example.test", + external_references={"calendar.email": "other@example.test"}, + ), + ) + + self.assertEqual((), records) + + def test_plan_retains_evidence_and_only_executes_preference_deletion(self) -> None: + records = self._records() + actions = self.provider.plan_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + records=records, + ) + + self.assertTrue({"retain", "manual_review"}.issubset({a.kind for a in actions})) + self.assertTrue( + any( + action.action_id == "calendar:retain:calendar_event:event-subject" + for action in actions + ) + ) + self.assertEqual( + {"calendar:delete:calendar_view_preference:preference-subject"}, + {action.action_id for action in actions if action.executable}, + ) + + def test_execution_is_revalidated_tenant_bound_and_idempotent(self) -> None: + action = self._preference_delete_action() + wrong_tenant = self.provider.execute_erasure( + self.session, + tenant_id="tenant-2", + subject=self.subject, + actions=(action,), + request_id="dsar-wrong-tenant", + ) + self.assertEqual("blocked", wrong_tenant[0].status) + + first = self.provider.execute_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + actions=(action,), + request_id="dsar-calendar-1", + ) + self.assertEqual("executed", first[0].status) + self.assertIsNone(self.session.get(CalendarViewPreference, self.preference.id)) + + repeated = self.provider.execute_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + actions=(action,), + request_id="dsar-calendar-1", + ) + self.assertEqual("unchanged", repeated[0].status) + + def test_execution_blocks_when_preference_owner_changed_after_planning( + self, + ) -> None: + action = self._preference_delete_action() + self.preference.user_id = "membership-2" + self.session.flush() + + result = self.provider.execute_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + actions=(action,), + request_id="dsar-stale", + ) + + self.assertEqual("blocked", result[0].status) + self.assertIsNotNone( + self.session.get(CalendarViewPreference, self.preference.id) + ) + + def test_core_workflow_discovers_active_provider_and_skips_it_when_disabled( + self, + ) -> None: + request = create_data_subject_request( + self.session, + tenant_id="tenant-1", + reference="DSAR-CALENDAR-1", + request_kind="access_and_erasure", + subject=self.subject, + purpose="Respond to an authorized privacy request.", + legal_basis="Article 15 and 17 GDPR", + due_at=None, + requested_by_account_id="privacy-officer", + ) + self.session.commit() + registry = _Registry(self.provider) + + search_data_subject_request( + self.session, + registry=registry, + row=request, + expected_revision=1, + ) + self.assertEqual("searched", request.status) + self.assertEqual(["calendar"], request.coverage["covered_modules"]) + plan_data_subject_erasure( + self.session, + registry=registry, + row=request, + expected_revision=2, + ) + executable_ids = [ + action["action_id"] + for action in request.erasure_plan["actions"] + if action["executable"] + ] + execute_data_subject_erasure( + self.session, + registry=registry, + row=request, + expected_revision=3, + action_ids=executable_ids, + ) + self.assertEqual("completed", request.status) + + disabled = create_data_subject_request( + self.session, + tenant_id="tenant-1", + reference="DSAR-CALENDAR-DISABLED", + request_kind="access", + subject=self.subject, + purpose="Verify disabled-module coverage.", + legal_basis="Article 15 GDPR", + due_at=None, + requested_by_account_id="privacy-officer", + ) + search_data_subject_request( + self.session, + registry=_Registry(self.provider, calendar_active=False), + row=disabled, + expected_revision=1, + ) + + self.assertEqual(0, disabled.search_result["record_count"]) + self.assertEqual( + [CALENDAR_DSAR_CAPABILITY], + disabled.coverage["inactive_provider_capabilities"], + ) + + def _records(self): + return self.provider.search_subject( + self.session, + tenant_id="tenant-1", + subject=self.subject, + ) + + def _preference_delete_action(self): + return next( + action + for action in self.provider.plan_erasure( + self.session, + tenant_id="tenant-1", + subject=self.subject, + records=self._records(), + ) + if action.resource_type == "calendar_view_preference" and action.executable + ) + + +if __name__ == "__main__": + unittest.main()