Files
govoplan-mail/src/govoplan_mail/backend/recovery.py
T

680 lines
25 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
import hashlib
from threading import Lock
from uuid import uuid4
from sqlalchemy import select
from govoplan_core.core.recovery import (
RecoveryGuaranteeError,
RecoveryMode,
RecoveryOperation,
RecoveryPlan,
RecoveryStatus,
)
from govoplan_core.core.recovery_runtime import (
DurableRecoveryOperation,
RecoveryOperationBusy,
RecoveryOperationStateConflict,
begin_durable_recovery_operation,
claim_durable_recovery_operation,
)
from govoplan_core.core.runtime_coordination import process_runtime_identity
from govoplan_core.db.session import get_database
from govoplan_mail.backend.db.models import (
MailBounceSource,
MailMailboxFolderIndex,
MailMailboxMessageIndex,
)
from govoplan_mail.backend.sending.imap import (
ImapFolderListResult,
ImapMailboxMessageListResult,
)
class MailRecoveryError(RuntimeError):
pass
class MailboxRefreshBusy(MailRecoveryError):
pass
_local_refresh_lock = Lock()
_local_refreshes: set[str] = set()
def _claim_local_refresh(key: str) -> bool:
with _local_refresh_lock:
if key in _local_refreshes:
return False
_local_refreshes.add(key)
return True
def _release_local_refresh(key: str) -> None:
with _local_refresh_lock:
_local_refreshes.discard(key)
def _digest(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _bounded_idempotency(kind: str, effect_id: str) -> str:
return f"mail-{kind}:{_digest(effect_id)}"
@dataclass(slots=True)
class ProviderEffectRecovery:
operation: DurableRecoveryOperation | None
operation_id: str
replayed: bool
kind: str
def reject(self, *, code: str, summary: str) -> None:
if self.operation is None:
return
self.operation.reject(
summary=summary,
evidence={
"verified": True,
"checks": {"provider_rejection": code},
"effect_kind": self.kind,
},
)
def unknown(self, *, code: str, summary: str) -> None:
if self.operation is None:
return
self.operation.unresolved(
status=RecoveryStatus.OUTCOME_UNKNOWN,
summary=summary,
evidence={"effect_started": True, "failure_code": code},
failure_summary=(
"Inspect the provider and reconcile the outcome before any retry"
),
)
def succeed_smtp(
self,
*,
accepted_count: int,
refused_recipients: dict[str, dict[str, int | str]],
) -> None:
if self.operation is None:
return
refused = [
{
"recipient_sha256": _digest(address.casefold()),
"classification": str(item.get("classification") or "unknown"),
"status_code": int(item.get("status_code") or 0),
}
for address, item in sorted(refused_recipients.items())
]
evidence = {
"verified": True,
"checks": {
"provider_returned": True,
"accepted_count": accepted_count,
"refused_count": len(refused),
},
"accepted_count": accepted_count,
"refused": refused,
}
if accepted_count > 0:
self.operation.succeed(evidence=evidence)
else:
self.operation.reject(
summary="SMTP definitively accepted no recipients",
evidence=evidence,
)
def succeed_imap(self, *, folder: str) -> None:
if self.operation is None:
return
self.operation.succeed(
evidence={
"verified": True,
"checks": {"provider_append_returned": True},
"folder_sha256": _digest(folder),
}
)
def begin_provider_effect_recovery(
*,
kind: str,
effect_id: str | None,
tenant_id: str,
profile_id: str,
message_bytes: bytes,
expected_transport_revision: str | None,
recipient_count: int | None = None,
folder: str | None = None,
resource_type: str | None = None,
resource_id: str | None = None,
) -> ProviderEffectRecovery | None:
"""Fence a Mail-owned provider mutation before any network effect.
``effect_id`` is optional only for compatibility with callers predating the
recovery contract. Current Mail and Campaign paths always supply one.
"""
clean_effect_id = str(effect_id or "").strip()
if not clean_effect_id:
return None
if kind not in {"smtp-delivery", "imap-append"}:
raise ValueError("Unsupported Mail provider recovery kind")
message_sha256 = hashlib.sha256(message_bytes).hexdigest()
request = {
"tenant_id": tenant_id,
"profile_id": profile_id,
"message_sha256": message_sha256,
"message_size_bytes": len(message_bytes),
"transport_revision": expected_transport_revision,
"recipient_count": recipient_count,
"folder_sha256": _digest(folder) if folder else None,
"effect_id_sha256": _digest(clean_effect_id),
}
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="mail",
operation_type=kind,
idempotency_key=_bounded_idempotency(kind, clean_effect_id),
request=request,
recovery_plan=RecoveryPlan(
mode=RecoveryMode.FORWARD_RECOVERY,
preconditions=(
"Mail authorized the profile and effective transport policy",
"the caller supplied a stable effect identifier",
"the request records message and address digests, never content or credentials",
),
forward_recovery_steps=(
"inspect provider evidence without repeating the effect",
"record whether the provider accepted the effect",
"retry only under a new deliberate attempt identifier when absence is proven",
),
verification_steps=(
"compare the provider outcome with the message digest and effect identifier",
"verify the caller's durable attempt state independently",
),
),
precondition_evidence=request,
lease_resource_key=f"mail:{kind}:{tenant_id}:{_digest(clean_effect_id)[:40]}",
lease_ttl_seconds=15 * 60,
resource_type=resource_type,
resource_id=resource_id,
metadata={"resources": ["postgresql", kind.split("-", 1)[0]]},
)
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
raise MailRecoveryError(
"This Mail provider effect already has an active or unresolved recovery record"
) from exc
except (RecoveryGuaranteeError, RuntimeError) as exc:
raise MailRecoveryError(
"The Mail recovery ledger is unavailable; no provider effect was started"
) from exc
return ProviderEffectRecovery(
operation=started.operation,
operation_id=started.operation_id,
replayed=started.replayed,
kind=kind,
)
def reconcile_outbox_provider_effect(
*,
command_id: str,
effect_occurred: bool,
evidence_reference: str,
user_id: str,
) -> bool:
"""Resolve the newest unknown SMTP effect for a durable outbox command."""
factory = get_database().SessionLocal
with factory() as evidence_session:
operation = evidence_session.scalar(
select(RecoveryOperation)
.where(
RecoveryOperation.module_id == "mail",
RecoveryOperation.operation_type == "smtp-delivery",
RecoveryOperation.resource_type == "mail_delivery_command",
RecoveryOperation.resource_id == command_id,
)
.order_by(RecoveryOperation.created_at.desc(), RecoveryOperation.id.desc())
.limit(1)
)
if operation is None:
return False
if effect_occurred and operation.status == RecoveryStatus.SUCCEEDED.value:
return True
if not effect_occurred and operation.status == RecoveryStatus.RECOVERED.value:
return True
if operation.status != RecoveryStatus.OUTCOME_UNKNOWN.value:
raise MailRecoveryError(
f"The Mail recovery record is already {operation.status}"
)
operation_id = operation.id
try:
recovery = claim_durable_recovery_operation(
factory,
identity=process_runtime_identity(),
operation_id=operation_id,
lease_ttl_seconds=15 * 60,
)
recovery.resolve_unknown(
effect_occurred=effect_occurred,
summary="An operator reconciled the SMTP provider outcome",
evidence={
"verified": True,
"checks": {
"provider_evidence_sha256": _digest(evidence_reference),
"reconciled_by_user_id": user_id,
},
"effect_occurred": effect_occurred,
},
)
except (RecoveryGuaranteeError, RecoveryOperationBusy) as exc:
raise MailRecoveryError(
"The Mail provider recovery record could not be reconciled"
) from exc
return True
@dataclass(slots=True)
class MailboxRefreshRecovery:
operation: DurableRecoveryOperation
tenant_id: str
profile_id: str
folder: str
local_key: str
def complete_folders(self, result: ImapFolderListResult) -> None:
expected = {item.name for item in result.folders}
with get_database().SessionLocal() as session:
rows = session.scalars(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
)
).all()
present = {row.folder for row in rows}
verified = present == expected
evidence = {
"verified": verified,
"checks": {
"expected_folder_count": len(expected),
"indexed_folder_count": len(present),
"expected_folders_sha256": _digest("\n".join(sorted(expected))),
"indexed_folders_sha256": _digest("\n".join(sorted(present))),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The IMAP folder read committed an incomplete mailbox index",
evidence=evidence,
failure_summary="Repeat the read-only index refresh under a new fence",
)
finally:
_release_local_refresh(self.local_key)
def complete_messages(self, result: ImapMailboxMessageListResult) -> None:
expected = {item.uid for item in result.messages}
with get_database().SessionLocal() as session:
rows = session.scalars(
select(MailMailboxMessageIndex).where(
MailMailboxMessageIndex.tenant_id == self.tenant_id,
MailMailboxMessageIndex.profile_id == self.profile_id,
MailMailboxMessageIndex.folder == result.folder,
MailMailboxMessageIndex.uid.in_(expected or {""}),
)
).all()
folder = session.scalar(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
MailMailboxFolderIndex.folder == result.folder,
)
)
present = {row.uid for row in rows}
verified = (
present == expected
and folder is not None
and folder.uidvalidity == result.uidvalidity
and folder.message_count == result.total_count
)
evidence = {
"verified": verified,
"checks": {
"expected_message_count": len(expected),
"indexed_message_count": len(present),
"uids_sha256": _digest("\n".join(sorted(expected))),
"uidvalidity_matches": bool(
folder is not None and folder.uidvalidity == result.uidvalidity
),
"total_count_matches": bool(
folder is not None and folder.message_count == result.total_count
),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The IMAP message read committed an incomplete mailbox index",
evidence=evidence,
failure_summary="Repeat the read-only index refresh under a new fence",
)
finally:
_release_local_refresh(self.local_key)
def complete_bootstrap(
self,
folders: ImapFolderListResult,
messages: ImapMailboxMessageListResult,
) -> None:
expected_folders = {item.name for item in folders.folders}
expected_uids = {item.uid for item in messages.messages}
with get_database().SessionLocal() as session:
folder_rows = session.scalars(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
)
).all()
message_rows = session.scalars(
select(MailMailboxMessageIndex).where(
MailMailboxMessageIndex.tenant_id == self.tenant_id,
MailMailboxMessageIndex.profile_id == self.profile_id,
MailMailboxMessageIndex.folder == messages.folder,
MailMailboxMessageIndex.uid.in_(expected_uids or {""}),
)
).all()
selected_folder = session.scalar(
select(MailMailboxFolderIndex).where(
MailMailboxFolderIndex.tenant_id == self.tenant_id,
MailMailboxFolderIndex.profile_id == self.profile_id,
MailMailboxFolderIndex.folder == messages.folder,
)
)
present_folders = {row.folder for row in folder_rows}
present_uids = {row.uid for row in message_rows}
verified = (
present_folders == expected_folders
and present_uids == expected_uids
and selected_folder is not None
and selected_folder.uidvalidity == messages.uidvalidity
and selected_folder.message_count == messages.total_count
)
evidence = {
"verified": verified,
"checks": {
"expected_folder_count": len(expected_folders),
"indexed_folder_count": len(present_folders),
"expected_message_count": len(expected_uids),
"indexed_message_count": len(present_uids),
"folder_set_matches": present_folders == expected_folders,
"message_window_matches": present_uids == expected_uids,
"uidvalidity_matches": bool(
selected_folder is not None
and selected_folder.uidvalidity == messages.uidvalidity
),
"total_count_matches": bool(
selected_folder is not None
and selected_folder.message_count == messages.total_count
),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The IMAP bootstrap committed an incomplete mailbox index",
evidence=evidence,
failure_summary="Repeat the read-only index refresh under a new fence",
)
finally:
_release_local_refresh(self.local_key)
def reject(self, *, summary: str, code: str) -> None:
try:
self.operation.reject(
summary=summary,
evidence={
"verified": True,
"checks": {
"provider_mutation": False,
"business_transaction_rolled_back": True,
"failure_code": code,
},
},
)
finally:
_release_local_refresh(self.local_key)
@dataclass(slots=True)
class BounceScanRecovery:
operation: DurableRecoveryOperation
source_id: str
local_key: str
def complete(self, *, highest_uid: int, uidvalidity: str | None) -> None:
with get_database().SessionLocal() as session:
source = session.get(MailBounceSource, self.source_id)
verified = bool(
source is not None
and source.highest_processed_uid == highest_uid
and source.uidvalidity == uidvalidity
and source.last_success_at is not None
)
evidence = {
"verified": verified,
"checks": {
"source_present": source is not None,
"cursor_matches": bool(
source is not None
and source.highest_processed_uid == highest_uid
),
"uidvalidity_matches": bool(
source is not None and source.uidvalidity == uidvalidity
),
"success_recorded": bool(
source is not None and source.last_success_at is not None
),
},
}
try:
if verified:
self.operation.succeed(evidence=evidence)
else:
self.operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The read-only bounce scan did not persist its verified cursor",
evidence=evidence,
failure_summary="Repeat the source scan under a new distributed fence",
)
finally:
_release_local_refresh(self.local_key)
def reject(self, *, code: str) -> None:
try:
self.operation.reject(
summary="The read-only bounce scan failed before its cursor committed",
evidence={
"verified": True,
"checks": {
"provider_mutation": False,
"business_transaction_rolled_back": True,
"failure_code": code,
},
},
)
finally:
_release_local_refresh(self.local_key)
def begin_mailbox_refresh_recovery(
*,
tenant_id: str,
profile_id: str,
folder: str,
purpose: str,
) -> MailboxRefreshRecovery:
folder_sha256 = _digest(folder)
refresh_id = str(uuid4())
local_key = f"mailbox:{tenant_id}:{profile_id}:{folder_sha256}"
if not _claim_local_refresh(local_key):
raise MailboxRefreshBusy(
"Another request in this runtime is already refreshing this mailbox index"
)
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="mail",
operation_type="mailbox-index-refresh",
idempotency_key=f"mailbox-refresh:{refresh_id}",
request={
"tenant_id": tenant_id,
"profile_id": profile_id,
"folder_sha256": folder_sha256,
"purpose": purpose,
},
recovery_plan=RecoveryPlan(
mode=RecoveryMode.ATOMIC,
preconditions=(
"the actor is authorized to read the selected Mail profile",
"the IMAP operation is read-only",
),
verification_steps=(
"reload the bounded mailbox index through an independent session",
"compare provider UID and folder metadata with committed cache rows",
),
),
precondition_evidence={
"profile_id": profile_id,
"folder_sha256": folder_sha256,
"provider_mutation": False,
},
lease_resource_key=f"mail:mailbox-refresh:{tenant_id}:{profile_id}:{folder_sha256[:24]}",
lease_ttl_seconds=5 * 60,
resource_type="mail_profile",
resource_id=profile_id,
metadata={"resources": ["postgresql", "imap"], "purpose": purpose},
)
except RecoveryOperationBusy as exc:
_release_local_refresh(local_key)
raise MailboxRefreshBusy(
"Another runtime is already refreshing this mailbox index"
) from exc
except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc:
_release_local_refresh(local_key)
raise MailRecoveryError(
"The mailbox recovery fence is unavailable; IMAP was not read"
) from exc
if started.operation is None:
_release_local_refresh(local_key)
raise MailRecoveryError("A mailbox refresh cannot replay a completed read")
return MailboxRefreshRecovery(
operation=started.operation,
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
local_key=local_key,
)
def begin_bounce_scan_recovery(
*,
tenant_id: str,
profile_id: str,
source_id: str,
folder: str,
) -> BounceScanRecovery:
folder_sha256 = _digest(folder)
local_key = f"bounce:{tenant_id}:{source_id}"
if not _claim_local_refresh(local_key):
raise MailboxRefreshBusy(
"Another request in this runtime is already scanning this bounce source"
)
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="mail",
operation_type="bounce-source-scan",
idempotency_key=f"bounce-scan:{uuid4()}",
request={
"tenant_id": tenant_id,
"profile_id": profile_id,
"source_id": source_id,
"folder_sha256": folder_sha256,
},
recovery_plan=RecoveryPlan(
mode=RecoveryMode.ATOMIC,
preconditions=(
"the configured source transport revision still matches",
"the IMAP scan does not alter provider mailbox state",
),
verification_steps=(
"reload the bounce source cursor independently",
"verify UIDVALIDITY, highest UID, and success timestamp",
),
),
precondition_evidence={
"source_id": source_id,
"folder_sha256": folder_sha256,
"provider_mutation": False,
},
lease_resource_key=f"mail:bounce-scan:{tenant_id}:{source_id}",
lease_ttl_seconds=15 * 60,
resource_type="mail_bounce_source",
resource_id=source_id,
metadata={"resources": ["postgresql", "imap"]},
)
except RecoveryOperationBusy as exc:
_release_local_refresh(local_key)
raise MailboxRefreshBusy(
"Another runtime is already scanning this bounce source"
) from exc
except (RecoveryOperationStateConflict, RecoveryGuaranteeError, RuntimeError) as exc:
_release_local_refresh(local_key)
raise MailRecoveryError(
"The bounce-source recovery fence is unavailable; IMAP was not read"
) from exc
if started.operation is None:
_release_local_refresh(local_key)
raise MailRecoveryError("A bounce-source scan cannot replay a completed read")
return BounceScanRecovery(
operation=started.operation,
source_id=source_id,
local_key=local_key,
)
__all__ = [
"MailRecoveryError",
"BounceScanRecovery",
"MailboxRefreshBusy",
"MailboxRefreshRecovery",
"ProviderEffectRecovery",
"begin_mailbox_refresh_recovery",
"begin_bounce_scan_recovery",
"begin_provider_effect_recovery",
"reconcile_outbox_provider_effect",
]