from __future__ import annotations from dataclasses import dataclass import hashlib from threading import Lock from uuid import uuid4 from sqlalchemy import select from govoplan_core.core.recovery import ( RecoveryGuaranteeError, RecoveryMode, RecoveryOperation, RecoveryPlan, RecoveryStatus, ) from govoplan_core.core.recovery_runtime import ( DurableRecoveryOperation, RecoveryOperationBusy, RecoveryOperationStateConflict, begin_durable_recovery_operation, claim_durable_recovery_operation, ) from govoplan_core.core.runtime_coordination import process_runtime_identity from govoplan_core.db.session import get_database from govoplan_mail.backend.db.models import ( MailBounceSource, MailMailboxFolderIndex, MailMailboxMessageIndex, ) from govoplan_mail.backend.sending.imap import ( ImapFolderListResult, ImapMailboxMessageListResult, ) class MailRecoveryError(RuntimeError): pass class MailboxRefreshBusy(MailRecoveryError): pass _local_refresh_lock = Lock() _local_refreshes: set[str] = set() def _claim_local_refresh(key: str) -> bool: with _local_refresh_lock: if key in _local_refreshes: return False _local_refreshes.add(key) return True def _release_local_refresh(key: str) -> None: with _local_refresh_lock: _local_refreshes.discard(key) def _digest(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() def _bounded_idempotency(kind: str, effect_id: str) -> str: return f"mail-{kind}:{_digest(effect_id)}" @dataclass(slots=True) class ProviderEffectRecovery: operation: DurableRecoveryOperation | None operation_id: str replayed: bool kind: str def reject(self, *, code: str, summary: str) -> None: if self.operation is None: return self.operation.reject( summary=summary, evidence={ "verified": True, "checks": {"provider_rejection": code}, "effect_kind": self.kind, }, ) def unknown(self, *, code: str, summary: str) -> None: if self.operation is None: return self.operation.unresolved( status=RecoveryStatus.OUTCOME_UNKNOWN, summary=summary, evidence={"effect_started": True, "failure_code": code}, failure_summary=( "Inspect the provider and reconcile the outcome before any retry" ), ) def succeed_smtp( self, *, accepted_count: int, refused_recipients: dict[str, dict[str, int | str]], ) -> None: if self.operation is None: return refused = [ { "recipient_sha256": _digest(address.casefold()), "classification": str(item.get("classification") or "unknown"), "status_code": int(item.get("status_code") or 0), } for address, item in sorted(refused_recipients.items()) ] evidence = { "verified": True, "checks": { "provider_returned": True, "accepted_count": accepted_count, "refused_count": len(refused), }, "accepted_count": accepted_count, "refused": refused, } if accepted_count > 0: self.operation.succeed(evidence=evidence) else: self.operation.reject( summary="SMTP definitively accepted no recipients", evidence=evidence, ) def succeed_imap(self, *, folder: str) -> None: if self.operation is None: return self.operation.succeed( evidence={ "verified": True, "checks": {"provider_append_returned": True}, "folder_sha256": _digest(folder), } ) def begin_provider_effect_recovery( *, kind: str, effect_id: str | None, tenant_id: str, profile_id: str, message_bytes: bytes, expected_transport_revision: str | None, recipient_count: int | None = None, folder: str | None = None, resource_type: str | None = None, resource_id: str | None = None, ) -> ProviderEffectRecovery | None: """Fence a Mail-owned provider mutation before any network effect. ``effect_id`` is optional only for compatibility with callers predating the recovery contract. Current Mail and Campaign paths always supply one. """ clean_effect_id = str(effect_id or "").strip() if not clean_effect_id: return None if kind not in {"smtp-delivery", "imap-append"}: raise ValueError("Unsupported Mail provider recovery kind") message_sha256 = hashlib.sha256(message_bytes).hexdigest() request = { "tenant_id": tenant_id, "profile_id": profile_id, "message_sha256": message_sha256, "message_size_bytes": len(message_bytes), "transport_revision": expected_transport_revision, "recipient_count": recipient_count, "folder_sha256": _digest(folder) if folder else None, "effect_id_sha256": _digest(clean_effect_id), } try: started = begin_durable_recovery_operation( get_database().SessionLocal, identity=process_runtime_identity(), module_id="mail", operation_type=kind, idempotency_key=_bounded_idempotency(kind, clean_effect_id), request=request, recovery_plan=RecoveryPlan( mode=RecoveryMode.FORWARD_RECOVERY, preconditions=( "Mail authorized the profile and effective transport policy", "the caller supplied a stable effect identifier", "the request records message and address digests, never content or credentials", ), forward_recovery_steps=( "inspect provider evidence without repeating the effect", "record whether the provider accepted the effect", "retry only under a new deliberate attempt identifier when absence is proven", ), verification_steps=( "compare the provider outcome with the message digest and effect identifier", "verify the caller's durable attempt state independently", ), ), precondition_evidence=request, lease_resource_key=f"mail:{kind}:{tenant_id}:{_digest(clean_effect_id)[:40]}", lease_ttl_seconds=15 * 60, resource_type=resource_type, resource_id=resource_id, metadata={"resources": ["postgresql", kind.split("-", 1)[0]]}, ) except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc: raise MailRecoveryError( "This Mail provider effect already has an active or unresolved recovery record" ) from exc except (RecoveryGuaranteeError, RuntimeError) as exc: raise MailRecoveryError( "The Mail recovery ledger is unavailable; no provider effect was started" ) from exc return ProviderEffectRecovery( operation=started.operation, operation_id=started.operation_id, replayed=started.replayed, kind=kind, ) def reconcile_outbox_provider_effect( *, command_id: str, effect_occurred: bool, evidence_reference: str, user_id: str, ) -> bool: """Resolve the newest unknown SMTP effect for a durable outbox command.""" factory = get_database().SessionLocal with factory() as evidence_session: operation = evidence_session.scalar( select(RecoveryOperation) .where( RecoveryOperation.module_id == "mail", RecoveryOperation.operation_type == "smtp-delivery", RecoveryOperation.resource_type == "mail_delivery_command", RecoveryOperation.resource_id == command_id, ) .order_by(RecoveryOperation.created_at.desc(), RecoveryOperation.id.desc()) .limit(1) ) if operation is None: return False if effect_occurred and operation.status == RecoveryStatus.SUCCEEDED.value: return True if not effect_occurred and operation.status == RecoveryStatus.RECOVERED.value: return True if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value: raise MailRecoveryError( f"The Mail recovery record is already {operation.status}" ) operation_id = operation.id try: recovery = claim_durable_recovery_operation( factory, identity=process_runtime_identity(), operation_id=operation_id, lease_ttl_seconds=15 * 60, ) recovery.resolve_unknown( effect_occurred=effect_occurred, summary="An operator reconciled the SMTP provider outcome", evidence={ "verified": True, "checks": { "provider_evidence_sha256": _digest(evidence_reference), "reconciled_by_user_id": user_id, }, "effect_occurred": effect_occurred, }, ) except (RecoveryGuaranteeError, RecoveryOperationBusy) as exc: raise MailRecoveryError( "The Mail provider recovery record could not be reconciled" ) from exc return True @dataclass(slots=True) class MailboxRefreshRecovery: operation: DurableRecoveryOperation tenant_id: str profile_id: str folder: str local_key: str def complete_folders(self, result: ImapFolderListResult) -> None: expected = {item.name for item in result.folders} with get_database().SessionLocal() as session: rows = session.scalars( select(MailMailboxFolderIndex).where( MailMailboxFolderIndex.tenant_id == self.tenant_id, MailMailboxFolderIndex.profile_id == self.profile_id, ) ).all() present = {row.folder for row in rows} verified = present == expected evidence = { "verified": verified, "checks": { "expected_folder_count": len(expected), "indexed_folder_count": len(present), "expected_folders_sha256": _digest("\n".join(sorted(expected))), "indexed_folders_sha256": _digest("\n".join(sorted(present))), }, } try: if verified: self.operation.succeed(evidence=evidence) else: self.operation.unresolved( status=RecoveryStatus.RECOVERY_REQUIRED, summary="The IMAP folder read committed an incomplete mailbox index", evidence=evidence, failure_summary="Repeat the read-only index refresh under a new fence", ) finally: _release_local_refresh(self.local_key) def complete_messages(self, result: ImapMailboxMessageListResult) -> None: expected = {item.uid for item in result.messages} with get_database().SessionLocal() as session: rows = session.scalars( select(MailMailboxMessageIndex).where( MailMailboxMessageIndex.tenant_id == self.tenant_id, MailMailboxMessageIndex.profile_id == self.profile_id, MailMailboxMessageIndex.folder == result.folder, MailMailboxMessageIndex.uid.in_(expected or {""}), ) ).all() folder = session.scalar( select(MailMailboxFolderIndex).where( MailMailboxFolderIndex.tenant_id == self.tenant_id, MailMailboxFolderIndex.profile_id == self.profile_id, MailMailboxFolderIndex.folder == result.folder, ) ) present = {row.uid for row in rows} verified = ( present == expected and folder is not None and folder.uidvalidity == result.uidvalidity and folder.message_count == result.total_count ) evidence = { "verified": verified, "checks": { "expected_message_count": len(expected), "indexed_message_count": len(present), "uids_sha256": _digest("\n".join(sorted(expected))), "uidvalidity_matches": bool( folder is not None and folder.uidvalidity == result.uidvalidity ), "total_count_matches": bool( folder is not None and folder.message_count == result.total_count ), }, } try: if verified: self.operation.succeed(evidence=evidence) else: self.operation.unresolved( status=RecoveryStatus.RECOVERY_REQUIRED, summary="The IMAP message read committed an incomplete mailbox index", evidence=evidence, failure_summary="Repeat the read-only index refresh under a new fence", ) finally: _release_local_refresh(self.local_key) def complete_bootstrap( self, folders: ImapFolderListResult, messages: ImapMailboxMessageListResult, ) -> None: expected_folders = {item.name for item in folders.folders} expected_uids = {item.uid for item in messages.messages} with get_database().SessionLocal() as session: folder_rows = session.scalars( select(MailMailboxFolderIndex).where( MailMailboxFolderIndex.tenant_id == self.tenant_id, MailMailboxFolderIndex.profile_id == self.profile_id, ) ).all() message_rows = session.scalars( select(MailMailboxMessageIndex).where( MailMailboxMessageIndex.tenant_id == self.tenant_id, MailMailboxMessageIndex.profile_id == self.profile_id, MailMailboxMessageIndex.folder == messages.folder, MailMailboxMessageIndex.uid.in_(expected_uids or {""}), ) ).all() selected_folder = session.scalar( select(MailMailboxFolderIndex).where( MailMailboxFolderIndex.tenant_id == self.tenant_id, MailMailboxFolderIndex.profile_id == self.profile_id, MailMailboxFolderIndex.folder == messages.folder, ) ) present_folders = {row.folder for row in folder_rows} present_uids = {row.uid for row in message_rows} verified = ( present_folders == expected_folders and present_uids == expected_uids and selected_folder is not None and selected_folder.uidvalidity == messages.uidvalidity and selected_folder.message_count == messages.total_count ) evidence = { "verified": verified, "checks": { "expected_folder_count": len(expected_folders), "indexed_folder_count": len(present_folders), "expected_message_count": len(expected_uids), "indexed_message_count": len(present_uids), "folder_set_matches": present_folders == expected_folders, "message_window_matches": present_uids == expected_uids, "uidvalidity_matches": bool( selected_folder is not None and selected_folder.uidvalidity == messages.uidvalidity ), "total_count_matches": bool( selected_folder is not None and selected_folder.message_count == messages.total_count ), }, } try: if verified: self.operation.succeed(evidence=evidence) else: self.operation.unresolved( status=RecoveryStatus.RECOVERY_REQUIRED, summary="The IMAP bootstrap committed an incomplete mailbox index", evidence=evidence, failure_summary="Repeat the read-only index refresh under a new fence", ) finally: _release_local_refresh(self.local_key) def reject(self, *, summary: str, code: str) -> None: try: self.operation.reject( summary=summary, evidence={ "verified": True, "checks": { "provider_mutation": False, "business_transaction_rolled_back": True, "failure_code": code, }, }, ) finally: _release_local_refresh(self.local_key) @dataclass(slots=True) class BounceScanRecovery: operation: DurableRecoveryOperation source_id: str local_key: str def complete(self, *, highest_uid: int, uidvalidity: str | None) -> None: with get_database().SessionLocal() as session: source = session.get(MailBounceSource, self.source_id) verified = bool( source is not None and source.highest_processed_uid == highest_uid and source.uidvalidity == uidvalidity and source.last_success_at is not None ) evidence = { "verified": verified, "checks": { "source_present": source is not None, "cursor_matches": bool( source is not None and source.highest_processed_uid == highest_uid ), "uidvalidity_matches": bool( source is not None and source.uidvalidity == uidvalidity ), "success_recorded": bool( source is not None and source.last_success_at is not None ), }, } try: if verified: self.operation.succeed(evidence=evidence) else: self.operation.unresolved( status=RecoveryStatus.RECOVERY_REQUIRED, summary="The read-only bounce scan did not persist its verified cursor", evidence=evidence, failure_summary="Repeat the source scan under a new distributed fence", ) finally: _release_local_refresh(self.local_key) def reject(self, *, code: str) -> None: try: self.operation.reject( summary="The read-only bounce scan failed before its cursor committed", evidence={ "verified": True, "checks": { "provider_mutation": False, "business_transaction_rolled_back": True, "failure_code": code, }, }, ) finally: _release_local_refresh(self.local_key) def begin_mailbox_refresh_recovery( *, tenant_id: str, profile_id: str, folder: str, purpose: str, ) -> MailboxRefreshRecovery: folder_sha256 = _digest(folder) refresh_id = str(uuid4()) local_key = f"mailbox:{tenant_id}:{profile_id}:{folder_sha256}" if not _claim_local_refresh(local_key): raise MailboxRefreshBusy( "Another request in this runtime is already refreshing this mailbox index" ) try: started = begin_durable_recovery_operation( get_database().SessionLocal, identity=process_runtime_identity(), module_id="mail", operation_type="mailbox-index-refresh", idempotency_key=f"mailbox-refresh:{refresh_id}", request={ "tenant_id": tenant_id, "profile_id": profile_id, "folder_sha256": folder_sha256, "purpose": purpose, }, recovery_plan=RecoveryPlan( mode=RecoveryMode.ATOMIC, preconditions=( "the actor is authorized to read the selected Mail profile", "the IMAP operation is read-only", ), verification_steps=( "reload the bounded mailbox index through an independent session", "compare provider UID and folder metadata with committed cache rows", ), ), precondition_evidence={ "profile_id": profile_id, "folder_sha256": folder_sha256, "provider_mutation": False, }, lease_resource_key=f"mail:mailbox-refresh:{tenant_id}:{profile_id}:{folder_sha256[:24]}", lease_ttl_seconds=5 * 60, resource_type="mail_profile", resource_id=profile_id, metadata={"resources": ["postgresql", "imap"], "purpose": purpose}, ) except RecoveryOperationBusy as exc: _release_local_refresh(local_key) raise MailboxRefreshBusy( "Another runtime is already refreshing this mailbox index" ) from exc except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc: _release_local_refresh(local_key) raise MailRecoveryError( "The mailbox recovery fence is unavailable; IMAP was not read" ) from exc if started.operation is None: _release_local_refresh(local_key) raise MailRecoveryError("A mailbox refresh cannot replay a completed read") return MailboxRefreshRecovery( operation=started.operation, tenant_id=tenant_id, profile_id=profile_id, folder=folder, local_key=local_key, ) def begin_bounce_scan_recovery( *, tenant_id: str, profile_id: str, source_id: str, folder: str, ) -> BounceScanRecovery: folder_sha256 = _digest(folder) local_key = f"bounce:{tenant_id}:{source_id}" if not _claim_local_refresh(local_key): raise MailboxRefreshBusy( "Another request in this runtime is already scanning this bounce source" ) try: started = begin_durable_recovery_operation( get_database().SessionLocal, identity=process_runtime_identity(), module_id="mail", operation_type="bounce-source-scan", idempotency_key=f"bounce-scan:{uuid4()}", request={ "tenant_id": tenant_id, "profile_id": profile_id, "source_id": source_id, "folder_sha256": folder_sha256, }, recovery_plan=RecoveryPlan( mode=RecoveryMode.ATOMIC, preconditions=( "the configured source transport revision still matches", "the IMAP scan does not alter provider mailbox state", ), verification_steps=( "reload the bounce source cursor independently", "verify UIDVALIDITY, highest UID, and success timestamp", ), ), precondition_evidence={ "source_id": source_id, "folder_sha256": folder_sha256, "provider_mutation": False, }, lease_resource_key=f"mail:bounce-scan:{tenant_id}:{source_id}", lease_ttl_seconds=15 * 60, resource_type="mail_bounce_source", resource_id=source_id, metadata={"resources": ["postgresql", "imap"]}, ) except RecoveryOperationBusy as exc: _release_local_refresh(local_key) raise MailboxRefreshBusy( "Another runtime is already scanning this bounce source" ) from exc except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc: _release_local_refresh(local_key) raise MailRecoveryError( "The bounce-source recovery fence is unavailable; IMAP was not read" ) from exc if started.operation is None: _release_local_refresh(local_key) raise MailRecoveryError("A bounce-source scan cannot replay a completed read") return BounceScanRecovery( operation=started.operation, source_id=source_id, local_key=local_key, ) __all__ = [ "MailRecoveryError", "BounceScanRecovery", "MailboxRefreshBusy", "MailboxRefreshRecovery", "ProviderEffectRecovery", "begin_mailbox_refresh_recovery", "begin_bounce_scan_recovery", "begin_provider_effect_recovery", "reconcile_outbox_provider_effect", ]