Fence Mail provider and mailbox recovery effects
This commit is contained in:
@@ -0,0 +1,679 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from threading import Lock
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryOperation,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
DurableRecoveryOperation,
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
claim_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_mail.backend.db.models import (
|
||||
MailBounceSource,
|
||||
MailMailboxFolderIndex,
|
||||
MailMailboxMessageIndex,
|
||||
)
|
||||
from govoplan_mail.backend.sending.imap import (
|
||||
ImapFolderListResult,
|
||||
ImapMailboxMessageListResult,
|
||||
)
|
||||
|
||||
|
||||
class MailRecoveryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class MailboxRefreshBusy(MailRecoveryError):
|
||||
pass
|
||||
|
||||
|
||||
_local_refresh_lock = Lock()
|
||||
_local_refreshes: set[str] = set()
|
||||
|
||||
|
||||
def _claim_local_refresh(key: str) -> bool:
|
||||
with _local_refresh_lock:
|
||||
if key in _local_refreshes:
|
||||
return False
|
||||
_local_refreshes.add(key)
|
||||
return True
|
||||
|
||||
|
||||
def _release_local_refresh(key: str) -> None:
|
||||
with _local_refresh_lock:
|
||||
_local_refreshes.discard(key)
|
||||
|
||||
|
||||
def _digest(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _bounded_idempotency(kind: str, effect_id: str) -> str:
|
||||
return f"mail-{kind}:{_digest(effect_id)}"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ProviderEffectRecovery:
|
||||
operation: DurableRecoveryOperation | None
|
||||
operation_id: str
|
||||
replayed: bool
|
||||
kind: str
|
||||
|
||||
def reject(self, *, code: str, summary: str) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
self.operation.reject(
|
||||
summary=summary,
|
||||
evidence={
|
||||
"verified": True,
|
||||
"checks": {"provider_rejection": code},
|
||||
"effect_kind": self.kind,
|
||||
},
|
||||
)
|
||||
|
||||
def unknown(self, *, code: str, summary: str) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary=summary,
|
||||
evidence={"effect_started": True, "failure_code": code},
|
||||
failure_summary=(
|
||||
"Inspect the provider and reconcile the outcome before any retry"
|
||||
),
|
||||
)
|
||||
|
||||
def succeed_smtp(
|
||||
self,
|
||||
*,
|
||||
accepted_count: int,
|
||||
refused_recipients: dict[str, dict[str, int | str]],
|
||||
) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
refused = [
|
||||
{
|
||||
"recipient_sha256": _digest(address.casefold()),
|
||||
"classification": str(item.get("classification") or "unknown"),
|
||||
"status_code": int(item.get("status_code") or 0),
|
||||
}
|
||||
for address, item in sorted(refused_recipients.items())
|
||||
]
|
||||
evidence = {
|
||||
"verified": True,
|
||||
"checks": {
|
||||
"provider_returned": True,
|
||||
"accepted_count": accepted_count,
|
||||
"refused_count": len(refused),
|
||||
},
|
||||
"accepted_count": accepted_count,
|
||||
"refused": refused,
|
||||
}
|
||||
if accepted_count > 0:
|
||||
self.operation.succeed(evidence=evidence)
|
||||
else:
|
||||
self.operation.reject(
|
||||
summary="SMTP definitively accepted no recipients",
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def succeed_imap(self, *, folder: str) -> None:
|
||||
if self.operation is None:
|
||||
return
|
||||
self.operation.succeed(
|
||||
evidence={
|
||||
"verified": True,
|
||||
"checks": {"provider_append_returned": True},
|
||||
"folder_sha256": _digest(folder),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def begin_provider_effect_recovery(
|
||||
*,
|
||||
kind: str,
|
||||
effect_id: str | None,
|
||||
tenant_id: str,
|
||||
profile_id: str,
|
||||
message_bytes: bytes,
|
||||
expected_transport_revision: str | None,
|
||||
recipient_count: int | None = None,
|
||||
folder: str | None = None,
|
||||
resource_type: str | None = None,
|
||||
resource_id: str | None = None,
|
||||
) -> ProviderEffectRecovery | None:
|
||||
"""Fence a Mail-owned provider mutation before any network effect.
|
||||
|
||||
``effect_id`` is optional only for compatibility with callers predating the
|
||||
recovery contract. Current Mail and Campaign paths always supply one.
|
||||
"""
|
||||
|
||||
clean_effect_id = str(effect_id or "").strip()
|
||||
if not clean_effect_id:
|
||||
return None
|
||||
if kind not in {"smtp-delivery", "imap-append"}:
|
||||
raise ValueError("Unsupported Mail provider recovery kind")
|
||||
message_sha256 = hashlib.sha256(message_bytes).hexdigest()
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"profile_id": profile_id,
|
||||
"message_sha256": message_sha256,
|
||||
"message_size_bytes": len(message_bytes),
|
||||
"transport_revision": expected_transport_revision,
|
||||
"recipient_count": recipient_count,
|
||||
"folder_sha256": _digest(folder) if folder else None,
|
||||
"effect_id_sha256": _digest(clean_effect_id),
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="mail",
|
||||
operation_type=kind,
|
||||
idempotency_key=_bounded_idempotency(kind, clean_effect_id),
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||
preconditions=(
|
||||
"Mail authorized the profile and effective transport policy",
|
||||
"the caller supplied a stable effect identifier",
|
||||
"the request records message and address digests, never content or credentials",
|
||||
),
|
||||
forward_recovery_steps=(
|
||||
"inspect provider evidence without repeating the effect",
|
||||
"record whether the provider accepted the effect",
|
||||
"retry only under a new deliberate attempt identifier when absence is proven",
|
||||
),
|
||||
verification_steps=(
|
||||
"compare the provider outcome with the message digest and effect identifier",
|
||||
"verify the caller's durable attempt state independently",
|
||||
),
|
||||
),
|
||||
precondition_evidence=request,
|
||||
lease_resource_key=f"mail:{kind}:{tenant_id}:{_digest(clean_effect_id)[:40]}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
metadata={"resources": ["postgresql", kind.split("-", 1)[0]]},
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
raise MailRecoveryError(
|
||||
"This Mail provider effect already has an active or unresolved recovery record"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
raise MailRecoveryError(
|
||||
"The Mail recovery ledger is unavailable; no provider effect was started"
|
||||
) from exc
|
||||
return ProviderEffectRecovery(
|
||||
operation=started.operation,
|
||||
operation_id=started.operation_id,
|
||||
replayed=started.replayed,
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
|
||||
def reconcile_outbox_provider_effect(
|
||||
*,
|
||||
command_id: str,
|
||||
effect_occurred: bool,
|
||||
evidence_reference: str,
|
||||
user_id: str,
|
||||
) -> bool:
|
||||
"""Resolve the newest unknown SMTP effect for a durable outbox command."""
|
||||
|
||||
factory = get_database().SessionLocal
|
||||
with factory() as evidence_session:
|
||||
operation = evidence_session.scalar(
|
||||
select(RecoveryOperation)
|
||||
.where(
|
||||
RecoveryOperation.module_id == "mail",
|
||||
RecoveryOperation.operation_type == "smtp-delivery",
|
||||
RecoveryOperation.resource_type == "mail_delivery_command",
|
||||
RecoveryOperation.resource_id == command_id,
|
||||
)
|
||||
.order_by(RecoveryOperation.created_at.desc(), RecoveryOperation.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if operation is None:
|
||||
return False
|
||||
if effect_occurred and operation.status == RecoveryStatus.SUCCEEDED.value:
|
||||
return True
|
||||
if not effect_occurred and operation.status == RecoveryStatus.RECOVERED.value:
|
||||
return True
|
||||
if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value:
|
||||
raise MailRecoveryError(
|
||||
f"The Mail recovery record is already {operation.status}"
|
||||
)
|
||||
operation_id = operation.id
|
||||
try:
|
||||
recovery = claim_durable_recovery_operation(
|
||||
factory,
|
||||
identity=process_runtime_identity(),
|
||||
operation_id=operation_id,
|
||||
lease_ttl_seconds=15 * 60,
|
||||
)
|
||||
recovery.resolve_unknown(
|
||||
effect_occurred=effect_occurred,
|
||||
summary="An operator reconciled the SMTP provider outcome",
|
||||
evidence={
|
||||
"verified": True,
|
||||
"checks": {
|
||||
"provider_evidence_sha256": _digest(evidence_reference),
|
||||
"reconciled_by_user_id": user_id,
|
||||
},
|
||||
"effect_occurred": effect_occurred,
|
||||
},
|
||||
)
|
||||
except (RecoveryGuaranteeError, RecoveryOperationBusy) as exc:
|
||||
raise MailRecoveryError(
|
||||
"The Mail provider recovery record could not be reconciled"
|
||||
) from exc
|
||||
return True
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class MailboxRefreshRecovery:
|
||||
operation: DurableRecoveryOperation
|
||||
tenant_id: str
|
||||
profile_id: str
|
||||
folder: str
|
||||
local_key: str
|
||||
|
||||
def complete_folders(self, result: ImapFolderListResult) -> None:
|
||||
expected = {item.name for item in result.folders}
|
||||
with get_database().SessionLocal() as session:
|
||||
rows = session.scalars(
|
||||
select(MailMailboxFolderIndex).where(
|
||||
MailMailboxFolderIndex.tenant_id == self.tenant_id,
|
||||
MailMailboxFolderIndex.profile_id == self.profile_id,
|
||||
)
|
||||
).all()
|
||||
present = {row.folder for row in rows}
|
||||
verified = present == expected
|
||||
evidence = {
|
||||
"verified": verified,
|
||||
"checks": {
|
||||
"expected_folder_count": len(expected),
|
||||
"indexed_folder_count": len(present),
|
||||
"expected_folders_sha256": _digest("\n".join(sorted(expected))),
|
||||
"indexed_folders_sha256": _digest("\n".join(sorted(present))),
|
||||
},
|
||||
}
|
||||
try:
|
||||
if verified:
|
||||
self.operation.succeed(evidence=evidence)
|
||||
else:
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The IMAP folder read committed an incomplete mailbox index",
|
||||
evidence=evidence,
|
||||
failure_summary="Repeat the read-only index refresh under a new fence",
|
||||
)
|
||||
finally:
|
||||
_release_local_refresh(self.local_key)
|
||||
|
||||
def complete_messages(self, result: ImapMailboxMessageListResult) -> None:
|
||||
expected = {item.uid for item in result.messages}
|
||||
with get_database().SessionLocal() as session:
|
||||
rows = session.scalars(
|
||||
select(MailMailboxMessageIndex).where(
|
||||
MailMailboxMessageIndex.tenant_id == self.tenant_id,
|
||||
MailMailboxMessageIndex.profile_id == self.profile_id,
|
||||
MailMailboxMessageIndex.folder == result.folder,
|
||||
MailMailboxMessageIndex.uid.in_(expected or {""}),
|
||||
)
|
||||
).all()
|
||||
folder = session.scalar(
|
||||
select(MailMailboxFolderIndex).where(
|
||||
MailMailboxFolderIndex.tenant_id == self.tenant_id,
|
||||
MailMailboxFolderIndex.profile_id == self.profile_id,
|
||||
MailMailboxFolderIndex.folder == result.folder,
|
||||
)
|
||||
)
|
||||
present = {row.uid for row in rows}
|
||||
verified = (
|
||||
present == expected
|
||||
and folder is not None
|
||||
and folder.uidvalidity == result.uidvalidity
|
||||
and folder.message_count == result.total_count
|
||||
)
|
||||
evidence = {
|
||||
"verified": verified,
|
||||
"checks": {
|
||||
"expected_message_count": len(expected),
|
||||
"indexed_message_count": len(present),
|
||||
"uids_sha256": _digest("\n".join(sorted(expected))),
|
||||
"uidvalidity_matches": bool(
|
||||
folder is not None and folder.uidvalidity == result.uidvalidity
|
||||
),
|
||||
"total_count_matches": bool(
|
||||
folder is not None and folder.message_count == result.total_count
|
||||
),
|
||||
},
|
||||
}
|
||||
try:
|
||||
if verified:
|
||||
self.operation.succeed(evidence=evidence)
|
||||
else:
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The IMAP message read committed an incomplete mailbox index",
|
||||
evidence=evidence,
|
||||
failure_summary="Repeat the read-only index refresh under a new fence",
|
||||
)
|
||||
finally:
|
||||
_release_local_refresh(self.local_key)
|
||||
|
||||
def complete_bootstrap(
|
||||
self,
|
||||
folders: ImapFolderListResult,
|
||||
messages: ImapMailboxMessageListResult,
|
||||
) -> None:
|
||||
expected_folders = {item.name for item in folders.folders}
|
||||
expected_uids = {item.uid for item in messages.messages}
|
||||
with get_database().SessionLocal() as session:
|
||||
folder_rows = session.scalars(
|
||||
select(MailMailboxFolderIndex).where(
|
||||
MailMailboxFolderIndex.tenant_id == self.tenant_id,
|
||||
MailMailboxFolderIndex.profile_id == self.profile_id,
|
||||
)
|
||||
).all()
|
||||
message_rows = session.scalars(
|
||||
select(MailMailboxMessageIndex).where(
|
||||
MailMailboxMessageIndex.tenant_id == self.tenant_id,
|
||||
MailMailboxMessageIndex.profile_id == self.profile_id,
|
||||
MailMailboxMessageIndex.folder == messages.folder,
|
||||
MailMailboxMessageIndex.uid.in_(expected_uids or {""}),
|
||||
)
|
||||
).all()
|
||||
selected_folder = session.scalar(
|
||||
select(MailMailboxFolderIndex).where(
|
||||
MailMailboxFolderIndex.tenant_id == self.tenant_id,
|
||||
MailMailboxFolderIndex.profile_id == self.profile_id,
|
||||
MailMailboxFolderIndex.folder == messages.folder,
|
||||
)
|
||||
)
|
||||
present_folders = {row.folder for row in folder_rows}
|
||||
present_uids = {row.uid for row in message_rows}
|
||||
verified = (
|
||||
present_folders == expected_folders
|
||||
and present_uids == expected_uids
|
||||
and selected_folder is not None
|
||||
and selected_folder.uidvalidity == messages.uidvalidity
|
||||
and selected_folder.message_count == messages.total_count
|
||||
)
|
||||
evidence = {
|
||||
"verified": verified,
|
||||
"checks": {
|
||||
"expected_folder_count": len(expected_folders),
|
||||
"indexed_folder_count": len(present_folders),
|
||||
"expected_message_count": len(expected_uids),
|
||||
"indexed_message_count": len(present_uids),
|
||||
"folder_set_matches": present_folders == expected_folders,
|
||||
"message_window_matches": present_uids == expected_uids,
|
||||
"uidvalidity_matches": bool(
|
||||
selected_folder is not None
|
||||
and selected_folder.uidvalidity == messages.uidvalidity
|
||||
),
|
||||
"total_count_matches": bool(
|
||||
selected_folder is not None
|
||||
and selected_folder.message_count == messages.total_count
|
||||
),
|
||||
},
|
||||
}
|
||||
try:
|
||||
if verified:
|
||||
self.operation.succeed(evidence=evidence)
|
||||
else:
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The IMAP bootstrap committed an incomplete mailbox index",
|
||||
evidence=evidence,
|
||||
failure_summary="Repeat the read-only index refresh under a new fence",
|
||||
)
|
||||
finally:
|
||||
_release_local_refresh(self.local_key)
|
||||
|
||||
def reject(self, *, summary: str, code: str) -> None:
|
||||
try:
|
||||
self.operation.reject(
|
||||
summary=summary,
|
||||
evidence={
|
||||
"verified": True,
|
||||
"checks": {
|
||||
"provider_mutation": False,
|
||||
"business_transaction_rolled_back": True,
|
||||
"failure_code": code,
|
||||
},
|
||||
},
|
||||
)
|
||||
finally:
|
||||
_release_local_refresh(self.local_key)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BounceScanRecovery:
|
||||
operation: DurableRecoveryOperation
|
||||
source_id: str
|
||||
local_key: str
|
||||
|
||||
def complete(self, *, highest_uid: int, uidvalidity: str | None) -> None:
|
||||
with get_database().SessionLocal() as session:
|
||||
source = session.get(MailBounceSource, self.source_id)
|
||||
verified = bool(
|
||||
source is not None
|
||||
and source.highest_processed_uid == highest_uid
|
||||
and source.uidvalidity == uidvalidity
|
||||
and source.last_success_at is not None
|
||||
)
|
||||
evidence = {
|
||||
"verified": verified,
|
||||
"checks": {
|
||||
"source_present": source is not None,
|
||||
"cursor_matches": bool(
|
||||
source is not None
|
||||
and source.highest_processed_uid == highest_uid
|
||||
),
|
||||
"uidvalidity_matches": bool(
|
||||
source is not None and source.uidvalidity == uidvalidity
|
||||
),
|
||||
"success_recorded": bool(
|
||||
source is not None and source.last_success_at is not None
|
||||
),
|
||||
},
|
||||
}
|
||||
try:
|
||||
if verified:
|
||||
self.operation.succeed(evidence=evidence)
|
||||
else:
|
||||
self.operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The read-only bounce scan did not persist its verified cursor",
|
||||
evidence=evidence,
|
||||
failure_summary="Repeat the source scan under a new distributed fence",
|
||||
)
|
||||
finally:
|
||||
_release_local_refresh(self.local_key)
|
||||
|
||||
def reject(self, *, code: str) -> None:
|
||||
try:
|
||||
self.operation.reject(
|
||||
summary="The read-only bounce scan failed before its cursor committed",
|
||||
evidence={
|
||||
"verified": True,
|
||||
"checks": {
|
||||
"provider_mutation": False,
|
||||
"business_transaction_rolled_back": True,
|
||||
"failure_code": code,
|
||||
},
|
||||
},
|
||||
)
|
||||
finally:
|
||||
_release_local_refresh(self.local_key)
|
||||
|
||||
|
||||
def begin_mailbox_refresh_recovery(
|
||||
*,
|
||||
tenant_id: str,
|
||||
profile_id: str,
|
||||
folder: str,
|
||||
purpose: str,
|
||||
) -> MailboxRefreshRecovery:
|
||||
folder_sha256 = _digest(folder)
|
||||
refresh_id = str(uuid4())
|
||||
local_key = f"mailbox:{tenant_id}:{profile_id}:{folder_sha256}"
|
||||
if not _claim_local_refresh(local_key):
|
||||
raise MailboxRefreshBusy(
|
||||
"Another request in this runtime is already refreshing this mailbox index"
|
||||
)
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="mail",
|
||||
operation_type="mailbox-index-refresh",
|
||||
idempotency_key=f"mailbox-refresh:{refresh_id}",
|
||||
request={
|
||||
"tenant_id": tenant_id,
|
||||
"profile_id": profile_id,
|
||||
"folder_sha256": folder_sha256,
|
||||
"purpose": purpose,
|
||||
},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.ATOMIC,
|
||||
preconditions=(
|
||||
"the actor is authorized to read the selected Mail profile",
|
||||
"the IMAP operation is read-only",
|
||||
),
|
||||
verification_steps=(
|
||||
"reload the bounded mailbox index through an independent session",
|
||||
"compare provider UID and folder metadata with committed cache rows",
|
||||
),
|
||||
),
|
||||
precondition_evidence={
|
||||
"profile_id": profile_id,
|
||||
"folder_sha256": folder_sha256,
|
||||
"provider_mutation": False,
|
||||
},
|
||||
lease_resource_key=f"mail:mailbox-refresh:{tenant_id}:{profile_id}:{folder_sha256[:24]}",
|
||||
lease_ttl_seconds=5 * 60,
|
||||
resource_type="mail_profile",
|
||||
resource_id=profile_id,
|
||||
metadata={"resources": ["postgresql", "imap"], "purpose": purpose},
|
||||
)
|
||||
except RecoveryOperationBusy as exc:
|
||||
_release_local_refresh(local_key)
|
||||
raise MailboxRefreshBusy(
|
||||
"Another runtime is already refreshing this mailbox index"
|
||||
) from exc
|
||||
except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
_release_local_refresh(local_key)
|
||||
raise MailRecoveryError(
|
||||
"The mailbox recovery fence is unavailable; IMAP was not read"
|
||||
) from exc
|
||||
if started.operation is None:
|
||||
_release_local_refresh(local_key)
|
||||
raise MailRecoveryError("A mailbox refresh cannot replay a completed read")
|
||||
return MailboxRefreshRecovery(
|
||||
operation=started.operation,
|
||||
tenant_id=tenant_id,
|
||||
profile_id=profile_id,
|
||||
folder=folder,
|
||||
local_key=local_key,
|
||||
)
|
||||
|
||||
|
||||
def begin_bounce_scan_recovery(
|
||||
*,
|
||||
tenant_id: str,
|
||||
profile_id: str,
|
||||
source_id: str,
|
||||
folder: str,
|
||||
) -> BounceScanRecovery:
|
||||
folder_sha256 = _digest(folder)
|
||||
local_key = f"bounce:{tenant_id}:{source_id}"
|
||||
if not _claim_local_refresh(local_key):
|
||||
raise MailboxRefreshBusy(
|
||||
"Another request in this runtime is already scanning this bounce source"
|
||||
)
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="mail",
|
||||
operation_type="bounce-source-scan",
|
||||
idempotency_key=f"bounce-scan:{uuid4()}",
|
||||
request={
|
||||
"tenant_id": tenant_id,
|
||||
"profile_id": profile_id,
|
||||
"source_id": source_id,
|
||||
"folder_sha256": folder_sha256,
|
||||
},
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.ATOMIC,
|
||||
preconditions=(
|
||||
"the configured source transport revision still matches",
|
||||
"the IMAP scan does not alter provider mailbox state",
|
||||
),
|
||||
verification_steps=(
|
||||
"reload the bounce source cursor independently",
|
||||
"verify UIDVALIDITY, highest UID, and success timestamp",
|
||||
),
|
||||
),
|
||||
precondition_evidence={
|
||||
"source_id": source_id,
|
||||
"folder_sha256": folder_sha256,
|
||||
"provider_mutation": False,
|
||||
},
|
||||
lease_resource_key=f"mail:bounce-scan:{tenant_id}:{source_id}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="mail_bounce_source",
|
||||
resource_id=source_id,
|
||||
metadata={"resources": ["postgresql", "imap"]},
|
||||
)
|
||||
except RecoveryOperationBusy as exc:
|
||||
_release_local_refresh(local_key)
|
||||
raise MailboxRefreshBusy(
|
||||
"Another runtime is already scanning this bounce source"
|
||||
) from exc
|
||||
except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc:
|
||||
_release_local_refresh(local_key)
|
||||
raise MailRecoveryError(
|
||||
"The bounce-source recovery fence is unavailable; IMAP was not read"
|
||||
) from exc
|
||||
if started.operation is None:
|
||||
_release_local_refresh(local_key)
|
||||
raise MailRecoveryError("A bounce-source scan cannot replay a completed read")
|
||||
return BounceScanRecovery(
|
||||
operation=started.operation,
|
||||
source_id=source_id,
|
||||
local_key=local_key,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MailRecoveryError",
|
||||
"BounceScanRecovery",
|
||||
"MailboxRefreshBusy",
|
||||
"MailboxRefreshRecovery",
|
||||
"ProviderEffectRecovery",
|
||||
"begin_mailbox_refresh_recovery",
|
||||
"begin_bounce_scan_recovery",
|
||||
"begin_provider_effect_recovery",
|
||||
"reconcile_outbox_provider_effect",
|
||||
]
|
||||
Reference in New Issue
Block a user