from __future__ import annotations import hashlib import json import uuid from collections.abc import Callable, Mapping from datetime import datetime, timedelta, timezone from typing import Any from sqlalchemy import and_, event, or_ from sqlalchemy.orm import Session from govoplan_calendar.backend.caldav import ( CalDAVNotFound, CalDAVObject, CalDAVPreconditionFailed, ) from govoplan_calendar.backend.db.models import ( CalendarEvent, CalendarOutboxOperation, CalendarSyncSource, ) from govoplan_calendar.backend.ical import events_to_ics, parse_vevents from govoplan_core.db.base import utcnow OUTBOX_PENDING_STATUSES = {"pending", "retry"} OUTBOX_ACTIVE_STATUSES = OUTBOX_PENDING_STATUSES | {"in_progress"} OUTBOX_TERMINAL_STATUSES = {"succeeded", "superseded", "cancelled", "conflict", "dead"} OUTBOX_UNRESOLVED_DESIRED_STATUSES = OUTBOX_ACTIVE_STATUSES | {"conflict", "dead"} OUTBOX_PURGEABLE_STATUSES = OUTBOX_TERMINAL_STATUSES - OUTBOX_UNRESOLVED_DESIRED_STATUSES OUTBOX_DEFAULT_MAX_ATTEMPTS = 8 OUTBOX_DEFAULT_TERMINAL_RETENTION_DAYS = 90 OUTBOX_LEASE_SECONDS = 300 OUTBOX_BACKOFF_BASE_SECONDS = 5 OUTBOX_BACKOFF_MAX_SECONDS = 3600 _AFTER_COMMIT_TENANTS = "govoplan_calendar_after_commit_outbox_tenants" def _source_configuration_snapshot(source: CalendarSyncSource) -> dict[str, str]: return { "calendar_id": source.calendar_id, "collection_url": source.collection_url, } def _operation_source_unavailable_reason( operation: CalendarOutboxOperation, source: CalendarSyncSource | None, ) -> str | None: if source is None: return "CalDAV source no longer exists" if source.tenant_id != operation.tenant_id: return "Calendar outbox source tenant does not match the operation tenant" if source.deleted_at is not None: return "CalDAV source was retired before this operation was delivered" if not source.sync_enabled: return "CalDAV source was disabled before this operation was delivered" if source.sync_direction != "two_way": return "CalDAV source was made inbound-only before this operation was delivered" metadata = operation.metadata_ or {} frozen_configuration = ( metadata.get("source_configuration") if isinstance(metadata, dict) else None ) if ( isinstance(frozen_configuration, dict) and frozen_configuration != _source_configuration_snapshot(source) ): return "CalDAV source endpoint or calendar changed before this operation was delivered" return None def _as_utc(value: datetime) -> datetime: if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) @event.listens_for(Session, "after_commit") def _dispatch_committed_calendar_outbox(session: Session) -> None: # ``after_commit`` also fires when a SAVEPOINT is released. The durable # row is not visible outside the outer transaction at that point, so keep # the wake-up registered until the root transaction commits. if session.in_nested_transaction(): return tenant_ids = tuple(session.info.pop(_AFTER_COMMIT_TENANTS, ())) for tenant_id in tenant_ids: _enqueue_celery_dispatch(tenant_id) @event.listens_for(Session, "after_rollback") def _discard_rolled_back_calendar_outbox(session: Session) -> None: # A nested rollback must not discard wake-ups registered by work in the # still-live outer transaction. A possibly superfluous wake-up after an # outer commit is harmless; the dispatcher simply finds no due row. if session.in_nested_transaction(): return session.info.pop(_AFTER_COMMIT_TENANTS, None) def _schedule_dispatch_after_commit(session: Session, tenant_id: str) -> None: tenants = session.info.setdefault(_AFTER_COMMIT_TENANTS, []) if tenant_id not in tenants: tenants.append(tenant_id) def _enqueue_celery_dispatch(tenant_id: str) -> None: try: from govoplan_core.celery_app import celery from govoplan_core.settings import settings if not getattr(settings, "celery_enabled", False): return celery.send_task( "govoplan.calendar.dispatch_outbox", args=[tenant_id, 50], queue="calendar", ) except Exception: # The committed outbox row is the source of truth. Broker/import # failures are recovered by the periodic dispatcher. return def calendar_payload_fingerprint(ics: str) -> str: """Return a semantic fingerprint resilient to harmless ICS formatting.""" try: components: list[dict[str, Any]] = [] for parsed in parse_vevents(ics): normalized = {key: value for key, value in parsed.items() if key != "raw_ics"} components.append(normalized) material = json.dumps(components, sort_keys=True, separators=(",", ":"), default=str) except Exception: material = "\n".join(line.rstrip() for line in ics.replace("\r\n", "\n").splitlines()).strip() return hashlib.sha256(material.encode("utf-8")).hexdigest() def _operation_idempotency_key( *, source: CalendarSyncSource, href: str, operation_kind: str, payload_fingerprint: str | None, expected_etag: str | None, generation: str, ) -> str: material = "\0".join( ( source.id, href, operation_kind, payload_fingerprint or "", expected_etag or "", generation, ) ) return hashlib.sha256(material.encode("utf-8")).hexdigest() def _event_generation(events: list[CalendarEvent], *, trigger_event: CalendarEvent | None) -> str: parts = [f"{item.id}:{item.sequence}" for item in sorted(events, key=lambda item: item.id)] if trigger_event is not None and trigger_event not in events: parts.append(f"trigger:{trigger_event.id}:{trigger_event.sequence}") return "|".join(parts) def _supersede_pending_resource_operations( session: Session, *, source_id: str, href: str, now: datetime, ) -> None: for operation in ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.source_id == source_id, CalendarOutboxOperation.resource_href == href, or_( and_( CalendarOutboxOperation.status == "pending", CalendarOutboxOperation.attempt_count == 0, CalendarOutboxOperation.last_attempt_at.is_(None), ), CalendarOutboxOperation.status.in_(("conflict", "dead")), ), ) .all() ): was_unattempted = operation.status == "pending" operation.status = "superseded" operation.completed_at = operation.completed_at or now if was_unattempted: operation.last_error = ( "Superseded by a newer desired state for the same CalDAV resource" ) def _set_event_outbox_state( event_model: CalendarEvent, *, source: CalendarSyncSource, href: str, operation: CalendarOutboxOperation, state: str, raw_ics: str | None, ) -> None: event_model.source_kind = "caldav" event_model.source_href = href if raw_ics is not None: event_model.raw_ics = raw_ics metadata = dict(event_model.metadata_ or {}) caldav_metadata = dict(metadata.get("caldav") or {}) caldav_metadata.update( { "source_id": source.id, "collection_url": source.collection_url, "href": href, "etag": event_model.etag, "external_state": state, "outbox_operation_id": operation.id, } ) metadata["caldav"] = caldav_metadata event_model.metadata_ = metadata def enqueue_caldav_put( session: Session, *, source: CalendarSyncSource, event_model: CalendarEvent, ) -> CalendarOutboxOperation: """Persist the exact desired remote resource in the caller transaction.""" from govoplan_calendar.backend.service import ( caldav_resource_events, caldav_resource_sort_key, generated_caldav_href, normalize_caldav_href, resource_etag, ) href = normalize_caldav_href( source.collection_url, event_model.source_href or generated_caldav_href(session, source=source, event=event_model), ) resource_events = caldav_resource_events(session, source=source, href=href) if event_model not in resource_events: resource_events.append(event_model) resource_events = sorted(resource_events, key=caldav_resource_sort_key) payload_ics = events_to_ics(resource_events) return enqueue_caldav_desired_state( session, source=source, trigger_event=event_model, href=href, resource_events=resource_events, payload_ics=payload_ics, expected_etag=resource_etag(resource_events), ) def enqueue_caldav_delete( session: Session, *, source: CalendarSyncSource, event_model: CalendarEvent, ) -> CalendarOutboxOperation | None: """Persist the desired state after deleting one VEVENT component.""" if not event_model.source_href: return None from govoplan_calendar.backend.service import ( caldav_resource_events, caldav_resource_sort_key, normalize_caldav_href, resource_etag, ) href = normalize_caldav_href(source.collection_url, event_model.source_href) remaining = [ item for item in caldav_resource_events(session, source=source, href=href) if item.id != event_model.id ] remaining = sorted(remaining, key=caldav_resource_sort_key) payload_ics = events_to_ics(remaining) if remaining else None expected_etag = event_model.etag or resource_etag(remaining) return enqueue_caldav_desired_state( session, source=source, trigger_event=event_model, href=href, resource_events=remaining, payload_ics=payload_ics, expected_etag=expected_etag, ) def enqueue_caldav_desired_state( session: Session, *, source: CalendarSyncSource, trigger_event: CalendarEvent | None, href: str, resource_events: list[CalendarEvent], payload_ics: str | None, expected_etag: str | None, ) -> CalendarOutboxOperation: now = utcnow() operation_kind = "put" if payload_ics is not None else "delete" fingerprint = calendar_payload_fingerprint(payload_ics) if payload_ics is not None else None idempotency_key = _operation_idempotency_key( source=source, href=href, operation_kind=operation_kind, payload_fingerprint=fingerprint, expected_etag=expected_etag, generation=_event_generation(resource_events, trigger_event=trigger_event), ) existing = ( session.query(CalendarOutboxOperation) .filter(CalendarOutboxOperation.idempotency_key == idempotency_key) .first() ) if existing is not None: return existing _supersede_pending_resource_operations(session, source_id=source.id, href=href, now=now) operation = CalendarOutboxOperation( tenant_id=source.tenant_id, source_id=source.id, event_id=trigger_event.id if trigger_event is not None else None, operation_kind=operation_kind, resource_href=href, payload_ics=payload_ics, payload_fingerprint=fingerprint, expected_etag=expected_etag, idempotency_key=idempotency_key, status="pending", attempt_count=0, max_attempts=OUTBOX_DEFAULT_MAX_ATTEMPTS, available_at=now, metadata_={ "event_ids": [item.id for item in resource_events], "overwrite": source.conflict_policy == "overwrite", "source_configuration": _source_configuration_snapshot(source), }, ) session.add(operation) session.flush() for item in resource_events: _set_event_outbox_state( item, source=source, href=href, operation=operation, state="queued", raw_ics=payload_ics, ) if trigger_event is not None and trigger_event not in resource_events: _set_event_outbox_state( trigger_event, source=source, href=href, operation=operation, state="queued_delete", raw_ics=None, ) session.flush() _schedule_dispatch_after_commit(session, source.tenant_id) return operation def list_calendar_outbox_operations( session: Session, *, tenant_id: str, status: str | None = None, source_id: str | None = None, limit: int = 100, ) -> list[CalendarOutboxOperation]: query = session.query(CalendarOutboxOperation).filter( CalendarOutboxOperation.tenant_id == tenant_id ) if status: query = query.filter(CalendarOutboxOperation.status == status) if source_id: query = query.filter(CalendarOutboxOperation.source_id == source_id) return query.order_by( CalendarOutboxOperation.created_at.desc(), CalendarOutboxOperation.id.desc(), ).limit(max(1, min(limit, 500))).all() def get_calendar_outbox_operation( session: Session, *, tenant_id: str, operation_id: str, ) -> CalendarOutboxOperation: operation = ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.tenant_id == tenant_id, CalendarOutboxOperation.id == operation_id, ) .first() ) if operation is None: raise ValueError("Calendar outbox operation not found") return operation def _lock_operation_source( session: Session, *, tenant_id: str, operation_id: str, ) -> tuple[CalendarOutboxOperation, CalendarSyncSource | None]: """Lock source then operation, matching the outbound worker lock order.""" peek = ( session.query(CalendarOutboxOperation.source_id) .filter( CalendarOutboxOperation.tenant_id == tenant_id, CalendarOutboxOperation.id == operation_id, ) .first() ) if peek is None: raise ValueError("Calendar outbox operation not found") source = ( session.query(CalendarSyncSource) .filter(CalendarSyncSource.id == peek[0]) .with_for_update() .one_or_none() ) operation = ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.tenant_id == tenant_id, CalendarOutboxOperation.id == operation_id, ) .with_for_update() .one() ) if source is not None and source.id != operation.source_id: raise ValueError("Calendar outbox source changed while acquiring the operation lock") return operation, source def _newer_resource_generation_exists( session: Session, operation: CalendarOutboxOperation, ) -> bool: return ( session.query(CalendarOutboxOperation.id) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.resource_href == operation.resource_href, or_( CalendarOutboxOperation.created_at > operation.created_at, and_( CalendarOutboxOperation.created_at == operation.created_at, CalendarOutboxOperation.id > operation.id, ), ), ) .first() is not None ) def _assert_latest_resource_generation( session: Session, operation: CalendarOutboxOperation, ) -> None: if _newer_resource_generation_exists(session, operation): raise ValueError( "This Calendar outbox operation is stale because a newer desired state exists; " "use the latest operation or make a new local edit" ) def _reclaim_expired_leases( session: Session, *, now: datetime, tenant_id: str | None = None, ) -> int: query = session.query(CalendarOutboxOperation).filter( CalendarOutboxOperation.status == "in_progress", CalendarOutboxOperation.lease_expires_at.is_not(None), CalendarOutboxOperation.lease_expires_at <= now, ) if tenant_id: query = query.filter(CalendarOutboxOperation.tenant_id == tenant_id) expired = query.with_for_update(skip_locked=True).all() for operation in expired: operation.status = "retry" operation.available_at = now operation.lease_token = None operation.lease_expires_at = None operation.last_error = "Worker lease expired; remote outcome will be reconciled before retry" return len(expired) def claim_next_calendar_outbox_operation( session: Session, *, tenant_id: str | None = None, now: datetime | None = None, lease_seconds: int = OUTBOX_LEASE_SECONDS, ) -> CalendarOutboxOperation | None: now = now or utcnow() _reclaim_expired_leases(session, now=now, tenant_id=tenant_id) query = session.query(CalendarOutboxOperation).filter( CalendarOutboxOperation.status.in_(sorted(OUTBOX_PENDING_STATUSES)), CalendarOutboxOperation.available_at <= now, ) if tenant_id: query = query.filter(CalendarOutboxOperation.tenant_id == tenant_id) candidates = query.order_by( CalendarOutboxOperation.available_at.asc(), CalendarOutboxOperation.created_at.asc(), CalendarOutboxOperation.id.asc(), ).with_for_update(skip_locked=True).limit(50).all() for operation in candidates: source_delivery_in_progress = ( session.query(CalendarOutboxOperation.id) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.status == "in_progress", ) .first() ) if source_delivery_in_progress is not None: # Delivery holds the source row lock across remote I/O. Leasing a # second href for the same source would only make its lease age # while it waits behind that lock. continue active_predecessor = ( session.query(CalendarOutboxOperation.id) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.resource_href == operation.resource_href, CalendarOutboxOperation.status.in_(sorted(OUTBOX_ACTIVE_STATUSES)), or_( CalendarOutboxOperation.created_at < operation.created_at, and_( CalendarOutboxOperation.created_at == operation.created_at, CalendarOutboxOperation.id < operation.id, ), ), ) .first() ) if active_predecessor is not None: continue succeeded_predecessor = ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.resource_href == operation.resource_href, CalendarOutboxOperation.status == "succeeded", CalendarOutboxOperation.completed_at.is_not(None), # Only repair the causal window where the newer row was # created while its predecessor was still completing. A # later inbound sync may legitimately have supplied a newer # expected ETag and must not be overwritten here. CalendarOutboxOperation.completed_at >= operation.created_at, or_( CalendarOutboxOperation.created_at < operation.created_at, and_( CalendarOutboxOperation.created_at == operation.created_at, CalendarOutboxOperation.id < operation.id, ), ), ) .order_by( CalendarOutboxOperation.created_at.desc(), CalendarOutboxOperation.id.desc(), ) .first() ) if succeeded_predecessor is not None: operation.expected_etag = ( succeeded_predecessor.remote_etag if succeeded_predecessor.operation_kind == "put" else None ) operation.status = "in_progress" operation.attempt_count += 1 operation.last_attempt_at = now operation.lease_token = str(uuid.uuid4()) operation.lease_expires_at = now + timedelta(seconds=max(30, lease_seconds)) session.flush() return operation session.flush() return None def _fetch_remote_object(client: object, href: str) -> CalDAVObject: fetch_state = getattr(client, "fetch_object_state", None) if callable(fetch_state): result = fetch_state(href) if isinstance(result, CalDAVObject): return result payload = client.fetch_object(href) return CalDAVObject(href=href, calendar_data=payload) def _remote_matches_operation(client: object, operation: CalendarOutboxOperation) -> tuple[bool, str | None]: try: remote = _fetch_remote_object(client, operation.resource_href) except CalDAVNotFound: return operation.operation_kind == "delete", None if operation.operation_kind == "delete": return False, remote.etag if not remote.calendar_data or not operation.payload_fingerprint: return False, remote.etag return calendar_payload_fingerprint(remote.calendar_data) == operation.payload_fingerprint, remote.etag def _operation_event_ids(operation: CalendarOutboxOperation) -> list[str]: metadata = operation.metadata_ or {} values = metadata.get("event_ids") if isinstance(metadata, dict) else None return [str(value) for value in values or []] def _event_points_to_operation(event_model: CalendarEvent, operation: CalendarOutboxOperation) -> bool: metadata = event_model.metadata_ or {} caldav = metadata.get("caldav") if isinstance(metadata, dict) else None return bool( isinstance(caldav, dict) and caldav.get("outbox_operation_id") == operation.id and caldav.get("source_id") == operation.source_id and caldav.get("href") == operation.resource_href and event_model.source_kind == "caldav" and event_model.source_href == operation.resource_href ) def _pointed_operation_events( session: Session, operation: CalendarOutboxOperation, ) -> list[CalendarEvent]: event_ids = set(_operation_event_ids(operation)) if operation.event_id: event_ids.add(operation.event_id) if not event_ids: return [] return [ event_model for event_model in ( session.query(CalendarEvent) .filter( CalendarEvent.tenant_id == operation.tenant_id, CalendarEvent.id.in_(sorted(event_ids)), ) .all() ) if _event_points_to_operation(event_model, operation) ] def _record_event_projection_change( session: Session, *, event_model: CalendarEvent, previous: dict[str, Any], ) -> None: from govoplan_calendar.backend.service import record_calendar_event_change record_calendar_event_change( session, event=event_model, operation="deleted" if event_model.deleted_at is not None else "updated", user_id=None, previous=previous, ) def _set_pointed_event_outbox_state( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, state: str, raw_ics: str | None, remote_etag: str | None = None, ) -> None: from govoplan_calendar.backend.service import calendar_event_change_payload for event_model in _pointed_operation_events(session, operation): previous = calendar_event_change_payload(event_model, prefix="previous_") if remote_etag: event_model.etag = remote_etag _set_event_outbox_state( event_model, source=source, href=operation.resource_href, operation=operation, state=state, raw_ics=raw_ics, ) _record_event_projection_change( session, event_model=event_model, previous=previous, ) def _complete_operation( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, remote_etag: str | None, reconciled: bool, ) -> None: now = utcnow() operation.status = "succeeded" operation.completed_at = now operation.reconciled_at = now if reconciled else operation.reconciled_at operation.remote_etag = remote_etag operation.last_error = None operation.lease_token = None operation.lease_expires_at = None source.last_status = "outbound_ok" source.last_error = None for stale_operation in ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.resource_href == operation.resource_href, CalendarOutboxOperation.status.in_(("conflict", "dead")), or_( CalendarOutboxOperation.created_at < operation.created_at, and_( CalendarOutboxOperation.created_at == operation.created_at, CalendarOutboxOperation.id < operation.id, ), ), ) .all() ): # A successfully delivered newer desired state makes older failed # snapshots audit history, not unresolved intent. stale_operation.status = "superseded" if operation.operation_kind == "put" and remote_etag: # A newer desired state may have been queued while this operation was # already in progress. Its snapshot can only know the predecessor # ETag, not the ETag produced by this PUT. Carry the causal ETag forward # so the newer PUT/DELETE is conditional on the state we just wrote. later_operations = ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.resource_href == operation.resource_href, CalendarOutboxOperation.id != operation.id, CalendarOutboxOperation.status.in_(sorted(OUTBOX_ACTIVE_STATUSES)), or_( CalendarOutboxOperation.created_at > operation.created_at, and_( CalendarOutboxOperation.created_at == operation.created_at, CalendarOutboxOperation.id > operation.id, ), ), ) .all() ) later_ids = {item.id for item in later_operations} for later_operation in later_operations: later_operation.expected_etag = remote_etag if later_ids: from govoplan_calendar.backend.service import calendar_event_change_payload for event_model in ( session.query(CalendarEvent) .filter( CalendarEvent.tenant_id == operation.tenant_id, CalendarEvent.source_href == operation.resource_href, ) .all() ): metadata = event_model.metadata_ or {} caldav = metadata.get("caldav") if isinstance(metadata, dict) else None if isinstance(caldav, dict) and caldav.get("outbox_operation_id") in later_ids: previous = calendar_event_change_payload(event_model, prefix="previous_") event_model.etag = remote_etag _record_event_projection_change( session, event_model=event_model, previous=previous, ) _set_pointed_event_outbox_state( session, operation=operation, source=source, state="synced", raw_ics=operation.payload_ics, remote_etag=remote_etag, ) session.flush() def _retry_delay_seconds(attempt_count: int) -> int: exponent = max(0, min(attempt_count - 1, 20)) return min(OUTBOX_BACKOFF_MAX_SECONDS, OUTBOX_BACKOFF_BASE_SECONDS * (2**exponent)) def _fail_operation( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, error: BaseException | str, terminal_status: str | None = None, ) -> None: now = utcnow() message = str(error)[:4000] if terminal_status: operation.status = terminal_status operation.completed_at = now elif operation.attempt_count >= operation.max_attempts: operation.status = "dead" operation.completed_at = now else: operation.status = "retry" operation.available_at = now + timedelta(seconds=_retry_delay_seconds(operation.attempt_count)) operation.last_error = message operation.lease_token = None operation.lease_expires_at = None if operation.status in {"dead", "conflict"}: source.last_status = "outbound_error" elif operation.status == "cancelled": source.last_status = "outbound_cancelled" else: source.last_status = "outbound_retry" source.last_error = message event_state = ( "failed" if operation.status in {"dead", "conflict"} else "cancelled" if operation.status == "cancelled" else "retry" ) _set_pointed_event_outbox_state( session, operation=operation, source=source, state=event_state, raw_ics=operation.payload_ics, ) session.flush() def _lock_leased_outbox_operation( session: Session, *, operation_id: str, lease_token: str, ) -> tuple[CalendarOutboxOperation, CalendarSyncSource | None]: operation = session.get(CalendarOutboxOperation, operation_id) if ( operation is None or operation.status != "in_progress" or operation.lease_token != lease_token ): raise ValueError("Calendar outbox lease is no longer valid") source = ( session.query(CalendarSyncSource) .filter(CalendarSyncSource.id == operation.source_id) .with_for_update() .one_or_none() ) operation = ( session.query(CalendarOutboxOperation) .filter(CalendarOutboxOperation.id == operation_id) .with_for_update() .one() ) if operation.status != "in_progress" or operation.lease_token != lease_token: raise ValueError("Calendar outbox lease is no longer valid") return operation, source def _cancel_undeliverable_operation( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource | None, reason: str, ) -> None: if source is not None and source.tenant_id == operation.tenant_id: _fail_operation( session, operation=operation, source=source, error=reason, terminal_status="cancelled", ) return operation.status = "cancelled" operation.completed_at = utcnow() operation.last_error = reason operation.lease_token = None operation.lease_expires_at = None session.flush() def _calendar_outbox_client( session: Session, *, source: CalendarSyncSource, client_factory: Callable[[Session, CalendarSyncSource], object] | None, ) -> object: if client_factory is not None: return client_factory(session, source) from govoplan_calendar.backend.service import caldav_client_for_source return caldav_client_for_source(session, source) def _try_remote_match( client: object, operation: CalendarOutboxOperation, ) -> tuple[bool, str | None] | None: try: return _remote_matches_operation(client, operation) except Exception: return None def _complete_if_remote_match_is_usable( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, remote_match: tuple[bool, str | None] | None, ) -> bool: if remote_match is None: return False matched, remote_etag = remote_match if not matched or (operation.operation_kind != "delete" and not remote_etag): return False _complete_operation( session, operation=operation, source=source, remote_etag=remote_etag, reconciled=True, ) return True def _execute_calendar_outbox_put( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, client: object, overwrite: bool, ) -> None: result = client.put_object( operation.resource_href, operation.payload_ics or "", etag=operation.expected_etag, create=not bool(operation.expected_etag), overwrite=overwrite, ) remote_etag = result.etag if remote_etag: _complete_operation( session, operation=operation, source=source, remote_etag=remote_etag, reconciled=False, ) return remote_match = _try_remote_match(client, operation) if _complete_if_remote_match_is_usable( session, operation=operation, source=source, remote_match=remote_match, ): return _fail_operation( session, operation=operation, source=source, error="CalDAV PUT succeeded without an ETag and could not be reconciled safely", ) def _execute_calendar_outbox_delete( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, client: object, overwrite: bool, ) -> None: if not operation.expected_etag and not overwrite: remote_match = _remote_matches_operation(client, operation) if _complete_if_remote_match_is_usable( session, operation=operation, source=source, remote_match=remote_match, ): return _fail_operation( session, operation=operation, source=source, error=( "CalDAV DELETE has no known ETag and the remote resource exists; " "refusing to delete an unknown remote version" ), terminal_status="conflict", ) return result = client.delete_object( operation.resource_href, etag=operation.expected_etag, overwrite=overwrite, ) _complete_operation( session, operation=operation, source=source, remote_etag=result.etag, reconciled=result.status == 404, ) def _handle_outbox_precondition_failure( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, client: object | None, error: CalDAVPreconditionFailed, ) -> None: remote_match = _try_remote_match(client, operation) if client is not None else None if _complete_if_remote_match_is_usable( session, operation=operation, source=source, remote_match=remote_match, ): return if remote_match is None: _fail_operation(session, operation=operation, source=source, error=error) return matched, _remote_etag = remote_match _fail_operation( session, operation=operation, source=source, error=( "Matching CalDAV resource has no usable ETag; retrying reconciliation" if matched else "CalDAV resource changed remotely; reconcile or sync before retrying" ), terminal_status=None if matched else "conflict", ) def _handle_outbox_delivery_failure( session: Session, *, operation: CalendarOutboxOperation, source: CalendarSyncSource, client: object | None, error: Exception, ) -> None: remote_match = _try_remote_match(client, operation) if client is not None else None if _complete_if_remote_match_is_usable( session, operation=operation, source=source, remote_match=remote_match, ): return _fail_operation(session, operation=operation, source=source, error=error) def execute_calendar_outbox_operation( session: Session, *, operation_id: str, lease_token: str, client_factory: Callable[[Session, CalendarSyncSource], object] | None = None, ) -> CalendarOutboxOperation: operation, source = _lock_leased_outbox_operation( session, operation_id=operation_id, lease_token=lease_token, ) unavailable_reason = _operation_source_unavailable_reason(operation, source) if unavailable_reason: _cancel_undeliverable_operation( session, operation=operation, source=source, reason=unavailable_reason, ) return operation if source is None: raise ValueError("Calendar outbox source is no longer available") operation_metadata = operation.metadata_ or {} overwrite = ( bool(operation_metadata.get("overwrite")) if isinstance(operation_metadata, dict) else False ) client: object | None = None try: client = _calendar_outbox_client( session, source=source, client_factory=client_factory, ) if operation.operation_kind == "put": _execute_calendar_outbox_put( session, operation=operation, source=source, client=client, overwrite=overwrite, ) else: _execute_calendar_outbox_delete( session, operation=operation, source=source, client=client, overwrite=overwrite, ) except CalDAVPreconditionFailed as exc: _handle_outbox_precondition_failure( session, operation=operation, source=source, client=client, error=exc, ) except Exception as exc: _handle_outbox_delivery_failure( session, operation=operation, source=source, client=client, error=exc, ) return operation def dispatch_calendar_outbox( session: Session, *, tenant_id: str | None = None, limit: int = 50, client_factory: Callable[[Session, CalendarSyncSource], object] | None = None, now: datetime | None = None, ) -> dict[str, object]: """Lease, deliver, and durably record up to ``limit`` operations.""" results: list[dict[str, object]] = [] for _index in range(max(1, min(limit, 500))): operation = claim_next_calendar_outbox_operation( session, tenant_id=tenant_id, now=now, ) if operation is None: session.commit() break operation_id = operation.id lease_token = operation.lease_token or "" # The lease must be visible before external I/O. session.commit() operation = execute_calendar_outbox_operation( session, operation_id=operation_id, lease_token=lease_token, client_factory=client_factory, ) results.append( { "id": operation.id, "status": operation.status, "attempt_count": operation.attempt_count, "last_error": operation.last_error, } ) session.commit() return { "processed": len(results), "succeeded": sum(item["status"] == "succeeded" for item in results), "retrying": sum(item["status"] == "retry" for item in results), "failed": sum(item["status"] in {"conflict", "dead", "cancelled"} for item in results), "operations": results, } def purge_terminal_calendar_outbox_operations( session: Session, *, retention_days: int, tenant_id: str | None = None, now: datetime | None = None, limit: int = 1000, ) -> int: """Delete bounded, resolved outbox history older than the configured window. ``conflict`` and ``dead`` are terminal delivery outcomes but still carry unresolved local desired state. They are intentionally excluded until an administrator retries, reconciles, or discards them. A retention of zero disables automatic cleanup. """ if retention_days < 0: raise ValueError("Calendar outbox terminal retention days must be zero or greater") if retention_days == 0: return 0 cutoff = (now or utcnow()) - timedelta(days=retention_days) query = session.query(CalendarOutboxOperation.id).filter( CalendarOutboxOperation.status.in_(sorted(OUTBOX_PURGEABLE_STATUSES)), CalendarOutboxOperation.completed_at.is_not(None), CalendarOutboxOperation.completed_at <= cutoff, ) if tenant_id: query = query.filter(CalendarOutboxOperation.tenant_id == tenant_id) operation_ids = [ operation_id for (operation_id,) in ( query.order_by( CalendarOutboxOperation.completed_at.asc(), CalendarOutboxOperation.id.asc(), ) .limit(max(1, min(limit, 10_000))) .all() ) ] if not operation_ids: return 0 deleted = ( session.query(CalendarOutboxOperation) .filter(CalendarOutboxOperation.id.in_(operation_ids)) .delete(synchronize_session=False) ) session.flush() return int(deleted) def retry_calendar_outbox_operation( session: Session, *, tenant_id: str, operation_id: str, ) -> CalendarOutboxOperation: operation, source = _lock_operation_source( session, tenant_id=tenant_id, operation_id=operation_id, ) if operation.status in {"succeeded", "in_progress", "superseded", "cancelled"}: raise ValueError(f"A {operation.status} Calendar outbox operation cannot be retried") _assert_latest_resource_generation(session, operation) unavailable_reason = _operation_source_unavailable_reason(operation, source) if unavailable_reason: raise ValueError(unavailable_reason) if source is None: raise RuntimeError("Calendar outbox source availability invariant was violated") operation.status = "pending" operation.attempt_count = 0 operation.available_at = utcnow() operation.completed_at = None operation.last_error = None operation.lease_token = None operation.lease_expires_at = None _set_pointed_event_outbox_state( session, operation=operation, source=source, state="queued", raw_ics=operation.payload_ics, ) session.flush() _schedule_dispatch_after_commit(session, tenant_id) return operation def reconcile_calendar_outbox_operation( session: Session, *, tenant_id: str, operation_id: str, client_factory: Callable[[Session, CalendarSyncSource], object] | None = None, ) -> CalendarOutboxOperation: operation, source = _lock_operation_source( session, tenant_id=tenant_id, operation_id=operation_id, ) if operation.status in {"succeeded", "superseded", "cancelled"}: raise ValueError(f"A {operation.status} Calendar outbox operation cannot be reconciled") if ( operation.status == "in_progress" and operation.lease_expires_at is not None and _as_utc(operation.lease_expires_at) > utcnow() ): raise ValueError("An actively leased Calendar outbox operation cannot be reconciled") _assert_latest_resource_generation(session, operation) unavailable_reason = _operation_source_unavailable_reason(operation, source) if unavailable_reason: raise ValueError(unavailable_reason) if source is None: raise RuntimeError("Calendar outbox source availability invariant was violated") if client_factory is None: from govoplan_calendar.backend.service import caldav_client_for_source client = caldav_client_for_source(session, source) else: client = client_factory(session, source) matched, remote_etag = _remote_matches_operation(client, operation) if matched and (operation.operation_kind == "delete" or remote_etag): _complete_operation( session, operation=operation, source=source, remote_etag=remote_etag, reconciled=True, ) elif matched: _fail_operation( session, operation=operation, source=source, error="Matching CalDAV resource has no usable ETag; retrying reconciliation", ) else: operation.reconciled_at = utcnow() _fail_operation( session, operation=operation, source=source, error="Remote CalDAV resource does not match the queued desired state", terminal_status="conflict", ) return operation def discard_calendar_outbox_operation( session: Session, *, tenant_id: str, operation_id: str, ) -> CalendarOutboxOperation: """Abandon local desired state so a later inbound sync may accept remote.""" operation, source = _lock_operation_source( session, tenant_id=tenant_id, operation_id=operation_id, ) if operation.status in {"succeeded", "superseded", "cancelled"}: raise ValueError(f"A {operation.status} Calendar outbox operation cannot be discarded") if ( operation.status == "in_progress" and operation.lease_expires_at is not None and _as_utc(operation.lease_expires_at) > utcnow() ): raise ValueError("An actively leased Calendar outbox operation cannot be discarded") _assert_latest_resource_generation(session, operation) unavailable_reason = _operation_source_unavailable_reason(operation, source) if unavailable_reason and ( source is None or source.tenant_id != operation.tenant_id ): raise ValueError(unavailable_reason) if source is None: raise RuntimeError("Calendar outbox source availability invariant was violated") now = utcnow() unresolved_chain = ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.source_id == operation.source_id, CalendarOutboxOperation.resource_href == operation.resource_href, CalendarOutboxOperation.status.in_(sorted(OUTBOX_UNRESOLVED_DESIRED_STATUSES)), or_( CalendarOutboxOperation.created_at < operation.created_at, and_( CalendarOutboxOperation.created_at == operation.created_at, CalendarOutboxOperation.id <= operation.id, ), ), ) .order_by( CalendarOutboxOperation.created_at.asc(), CalendarOutboxOperation.id.asc(), ) .with_for_update() .all() ) if any( item.status == "in_progress" and item.lease_expires_at is not None and _as_utc(item.lease_expires_at) > now for item in unresolved_chain ): raise ValueError( "The Calendar resource has an actively leased predecessor and cannot be discarded" ) message = ( "Local desired state was discarded by an administrator; accept remote on next sync" ) for item in unresolved_chain: item.status = "cancelled" item.completed_at = now item.reconciled_at = now item.last_error = message item.lease_token = None item.lease_expires_at = None _set_pointed_event_outbox_state( session, operation=item, source=source, state="discarded", raw_ics=item.payload_ics, ) source.last_status = "outbound_discarded" source.last_error = message # Incremental sync might not report an unchanged remote object. Force the # next source run to perform a full collection reconciliation. source.sync_token = None source.next_sync_at = now session.flush() return operation def cancel_calendar_outbox_for_source( session: Session, *, source_id: str, reason: str, ) -> int: now = utcnow() operations = ( session.query(CalendarOutboxOperation) .filter( CalendarOutboxOperation.source_id == source_id, CalendarOutboxOperation.status.in_(sorted(OUTBOX_UNRESOLVED_DESIRED_STATUSES)), ) .all() ) source = session.get(CalendarSyncSource, source_id) for operation in operations: operation.status = "cancelled" operation.completed_at = now operation.last_error = reason operation.lease_token = None operation.lease_expires_at = None if source is not None and source.tenant_id == operation.tenant_id: _set_pointed_event_outbox_state( session, operation=operation, source=source, state="cancelled", raw_ics=operation.payload_ics, ) return len(operations) def calendar_outbox_has_live_lease( session: Session, *, source_id: str, now: datetime | None = None, ) -> bool: now = now or utcnow() return ( session.query(CalendarOutboxOperation.id) .filter( CalendarOutboxOperation.source_id == source_id, CalendarOutboxOperation.status == "in_progress", CalendarOutboxOperation.lease_expires_at > now, ) .first() is not None ) def calendar_outbox_has_active_desired_state( session: Session, *, source_id: str, href: str, ) -> bool: latest_status = ( session.query(CalendarOutboxOperation.status) .filter( CalendarOutboxOperation.source_id == source_id, CalendarOutboxOperation.resource_href == href, ) .order_by( CalendarOutboxOperation.created_at.desc(), CalendarOutboxOperation.id.desc(), ) .first() ) return bool( latest_status is not None and latest_status[0] in OUTBOX_UNRESOLVED_DESIRED_STATUSES ) def calendar_outbox_has_unresolved_desired_state( session: Session, *, source_id: str, ) -> bool: rows = ( session.query( CalendarOutboxOperation.resource_href, CalendarOutboxOperation.status, ) .filter( CalendarOutboxOperation.source_id == source_id, ) .order_by( CalendarOutboxOperation.resource_href.asc(), CalendarOutboxOperation.created_at.desc(), CalendarOutboxOperation.id.desc(), ) .all() ) seen_hrefs: set[str] = set() for href, status in rows: if href in seen_hrefs: continue seen_hrefs.add(href) if status in OUTBOX_UNRESOLVED_DESIRED_STATUSES: return True return False def calendar_outbox_operation_response(operation: CalendarOutboxOperation) -> dict[str, Any]: return { "id": operation.id, "tenant_id": operation.tenant_id, "source_id": operation.source_id, "event_id": operation.event_id, "operation_kind": operation.operation_kind, "resource_href": operation.resource_href, "payload_fingerprint": operation.payload_fingerprint, "expected_etag": operation.expected_etag, "idempotency_key": operation.idempotency_key, "status": operation.status, "attempt_count": operation.attempt_count, "max_attempts": operation.max_attempts, "available_at": operation.available_at, "last_attempt_at": operation.last_attempt_at, "lease_expires_at": operation.lease_expires_at, "completed_at": operation.completed_at, "reconciled_at": operation.reconciled_at, "remote_etag": operation.remote_etag, "last_error": operation.last_error, "created_at": operation.created_at, "updated_at": operation.updated_at, "metadata": dict(operation.metadata_ or {}), } class SqlCalendarOutboxProvider: def __init__( self, *, terminal_retention_days: int = OUTBOX_DEFAULT_TERMINAL_RETENTION_DAYS, ) -> None: if isinstance(terminal_retention_days, bool): raise ValueError("Calendar outbox terminal retention days must be an integer") try: normalized_retention_days = int(terminal_retention_days) except (TypeError, ValueError) as exc: raise ValueError("Calendar outbox terminal retention days must be an integer") from exc if normalized_retention_days < 0: raise ValueError("Calendar outbox terminal retention days must be zero or greater") self.terminal_retention_days = normalized_retention_days def dispatch_due( self, session: object, *, tenant_id: str | None = None, limit: int = 50, ) -> Mapping[str, object]: if not isinstance(session, Session): raise TypeError("Calendar outbox requires a SQLAlchemy Session") result = dispatch_calendar_outbox(session, tenant_id=tenant_id, limit=limit) result["purged"] = purge_terminal_calendar_outbox_operations( session, retention_days=self.terminal_retention_days, tenant_id=tenant_id, ) return result __all__ = [ "OUTBOX_ACTIVE_STATUSES", "OUTBOX_DEFAULT_MAX_ATTEMPTS", "OUTBOX_DEFAULT_TERMINAL_RETENTION_DAYS", "OUTBOX_PURGEABLE_STATUSES", "OUTBOX_TERMINAL_STATUSES", "OUTBOX_UNRESOLVED_DESIRED_STATUSES", "SqlCalendarOutboxProvider", "calendar_outbox_has_active_desired_state", "calendar_outbox_has_live_lease", "calendar_outbox_has_unresolved_desired_state", "calendar_outbox_operation_response", "calendar_payload_fingerprint", "cancel_calendar_outbox_for_source", "claim_next_calendar_outbox_operation", "dispatch_calendar_outbox", "discard_calendar_outbox_operation", "enqueue_caldav_delete", "enqueue_caldav_put", "get_calendar_outbox_operation", "list_calendar_outbox_operations", "purge_terminal_calendar_outbox_operations", "reconcile_calendar_outbox_operation", "retry_calendar_outbox_operation", ]