intermittent commit

src/govoplan_mail/backend/sending/imap.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import imaplib
+import logging
 import re
 import socket
 import ssl
@@ -21,6 +22,8 @@ from govoplan_mail.backend.dev.mock_mailbox import (
     record_imap_append,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class ImapConfigurationError(ValueError):
     """Raised when IMAP settings are incomplete or inconsistent."""
@@ -70,6 +73,10 @@ class ImapMailboxAttachmentInfo:
     size_bytes: int
 
 
+def _log_imap_cleanup_failure(action: str, exc: BaseException) -> None:
+    logger.debug("IMAP cleanup failed while %s: %s", action, exc, exc_info=True)
+
+
 @dataclass(frozen=True, slots=True)
 class ImapMailboxMessageSummary:
     uid: str
@@ -119,6 +126,12 @@ class ImapMailboxMessageListResult:
     cursor_reset: bool = False
 
 
+@dataclass(frozen=True, slots=True)
+class ImapMailboxBootstrapResult:
+    folders: ImapFolderListResult
+    messages: ImapMailboxMessageListResult
+
+
 @dataclass(frozen=True, slots=True)
 class ImapMailboxMessageResult:
     host: str
@@ -170,8 +183,8 @@ def _open_imap(config: ImapConfig) -> imaplib.IMAP4:
     except Exception:
         try:
             client.logout()  # type: ignore[possibly-undefined]
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            _log_imap_cleanup_failure("opening connection", cleanup_exc)
         raise
 
 
@@ -306,58 +319,200 @@ def test_imap_login(*, imap_config: ImapConfig) -> ImapLoginTestResult:
     finally:
         try:
             client.logout()
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            _log_imap_cleanup_failure("testing login", cleanup_exc)
 
 
-def list_imap_folders(*, imap_config: ImapConfig) -> ImapFolderListResult:
+def _mock_imap_folders(*, imap_config: ImapConfig) -> ImapFolderListResult:
+    host, port = _require_imap_config(imap_config)
+    records = list_records(limit=500)
+    folders = []
+    for item in MOCK_IMAP_FOLDERS:
+        name = str(item["name"])
+        count = sum(1 for record in records if _mock_folder_matches(record, name))
+        folders.append(ImapMailboxInfo(name=name, flags=list(item.get("flags") or []), message_count=count, unseen_count=None))
+    return ImapFolderListResult(
+        host=host,
+        port=port,
+        security=imap_config.security.value,
+        folders=folders,
+        detected_sent_folder="Sent",
+    )
+
+
+def _list_imap_folders_on_client(
+    client: imaplib.IMAP4,
+    *,
+    host: str,
+    port: int,
+    security: str,
+    include_status: bool,
+) -> ImapFolderListResult:
+    typ, data = client.list()
+    if typ != "OK":
+        raise ImapAppendError(f"IMAP folder listing failed: {data!r}", temporary=True)
+
+    parsed: list[tuple[str, set[str]]] = []
+    folders: list[ImapMailboxInfo] = []
+    for item in data or []:
+        extracted = _extract_mailbox_name(item)
+        if not extracted:
+            continue
+        name, flags = extracted
+        parsed.append((name, flags))
+        message_count, unseen_count = (
+            (None, None)
+            if not include_status or _has_folder_flag(flags, "noselect")
+            else _imap_folder_status(client, name)
+        )
+        folders.append(ImapMailboxInfo(name=name, flags=sorted(flags), message_count=message_count, unseen_count=unseen_count))
+
+    return ImapFolderListResult(
+        host=host,
+        port=port,
+        security=security,
+        folders=folders,
+        detected_sent_folder=_detect_sent_folder(parsed),
+    )
+
+
+def list_imap_folders(*, imap_config: ImapConfig, include_status: bool = True) -> ImapFolderListResult:
     """Return folders visible through IMAP LIST and the best sent-folder guess."""
 
     host, port = _require_imap_config(imap_config)
     if is_mock_imap_host(imap_config.host):
-        records = list_records(limit=500)
-        folders = []
-        for item in MOCK_IMAP_FOLDERS:
-            name = str(item["name"])
-            count = sum(1 for record in records if _mock_folder_matches(record, name))
-            folders.append(ImapMailboxInfo(name=name, flags=list(item.get("flags") or []), message_count=count, unseen_count=None))
-        return ImapFolderListResult(
-            host=host,
-            port=port,
-            security=imap_config.security.value,
-            folders=folders,
-            detected_sent_folder="Sent",
-        )
+        return _mock_imap_folders(imap_config=imap_config)
 
     client = _open_imap(imap_config)
     try:
-        typ, data = client.list()
-        if typ != "OK":
-            raise ImapAppendError(f"IMAP folder listing failed: {data!r}", temporary=True)
-
-        parsed: list[tuple[str, set[str]]] = []
-        folders: list[ImapMailboxInfo] = []
-        for item in data or []:
-            extracted = _extract_mailbox_name(item)
-            if not extracted:
-                continue
-            name, flags = extracted
-            parsed.append((name, flags))
-            message_count, unseen_count = (None, None) if _has_folder_flag(flags, "noselect") else _imap_folder_status(client, name)
-            folders.append(ImapMailboxInfo(name=name, flags=sorted(flags), message_count=message_count, unseen_count=unseen_count))
-
-        return ImapFolderListResult(
+        return _list_imap_folders_on_client(
+            client,
             host=host,
             port=port,
             security=imap_config.security.value,
-            folders=folders,
-            detected_sent_folder=_detect_sent_folder(parsed),
+            include_status=include_status,
         )
     finally:
         try:
             client.logout()
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            _log_imap_cleanup_failure("listing folders", cleanup_exc)
+
+
+def _preferred_mailbox_folder(folders: list[ImapMailboxInfo], requested: str | None, detected_sent_folder: str | None) -> str:
+    folder_names = {folder.name for folder in folders}
+    requested = (requested or "").strip()
+    if requested and requested in folder_names:
+        return requested
+    if "INBOX" in folder_names:
+        return "INBOX"
+    if detected_sent_folder and detected_sent_folder in folder_names:
+        return detected_sent_folder
+    return folders[0].name if folders else (requested or "INBOX")
+
+
+def load_imap_mailbox_bootstrap(
+    *,
+    imap_config: ImapConfig,
+    folder: str = "INBOX",
+    limit: int = 50,
+    offset: int = 0,
+    include_folder_status: bool = False,
+) -> ImapMailboxBootstrapResult:
+    """Load folders and one message page through a single IMAP connection."""
+
+    host, port = _require_imap_config(imap_config)
+    if is_mock_imap_host(imap_config.host):
+        folders = _mock_imap_folders(imap_config=imap_config)
+        selected_folder = _preferred_mailbox_folder(folders.folders, folder, folders.detected_sent_folder)
+        messages = list_imap_messages(imap_config=imap_config, folder=selected_folder, limit=limit, offset=offset)
+        return ImapMailboxBootstrapResult(folders=folders, messages=messages)
+
+    client = _open_imap(imap_config)
+    try:
+        folders = _list_imap_folders_on_client(
+            client,
+            host=host,
+            port=port,
+            security=imap_config.security.value,
+            include_status=include_folder_status,
+        )
+        selected_folder = _preferred_mailbox_folder(folders.folders, folder, folders.detected_sent_folder)
+        selected_folder, limit, offset = _normalize_mailbox_page(folder=selected_folder, limit=limit, offset=offset)
+        messages = _list_imap_messages_on_client(
+            client,
+            host=host,
+            port=port,
+            security=imap_config.security.value,
+            folder=selected_folder,
+            limit=limit,
+            offset=offset,
+            after_uid=None,
+            expected_uidvalidity=None,
+        )
+        return ImapMailboxBootstrapResult(folders=folders, messages=messages)
+    finally:
+        try:
+            client.logout()
+        except Exception as cleanup_exc:
+            _log_imap_cleanup_failure("bootstrapping mailbox", cleanup_exc)
+
+
+def _list_imap_messages_on_client(
+    client: imaplib.IMAP4,
+    *,
+    host: str,
+    port: int,
+    security: str,
+    folder: str,
+    limit: int,
+    offset: int,
+    after_uid: str | None,
+    expected_uidvalidity: str | None,
+) -> ImapMailboxMessageListResult:
+    total_count, uidvalidity = _select_readonly(client, folder)
+    if after_uid is None and expected_uidvalidity is None:
+        page_sequences = _paged_descending_sequences(total_count, offset=offset, limit=limit)
+        messages = _fetch_message_summaries_by_sequence(client, page_sequences, folder)
+        return ImapMailboxMessageListResult(
+            host=host,
+            port=port,
+            security=security,
+            folder=folder,
+            messages=messages,
+            total_count=total_count,
+            offset=offset,
+            limit=limit,
+            uidvalidity=None,
+            cursor_reset=False,
+        )
+    uids = _search_message_uids(client)
+    cursor_reset = False
+    effective_after_uid = after_uid
+    effective_offset = offset
+    if expected_uidvalidity and expected_uidvalidity != uidvalidity:
+        effective_after_uid = None
+        effective_offset = 0
+        cursor_reset = True
+    page_uids, effective_offset, anchor_missing = _paged_descending_uids(
+        uids,
+        offset=effective_offset,
+        limit=limit,
+        after_uid=effective_after_uid,
+    )
+    messages = _fetch_message_summaries_by_uid(client, page_uids, folder)
+    return ImapMailboxMessageListResult(
+        host=host,
+        port=port,
+        security=security,
+        folder=folder,
+        messages=messages,
+        total_count=len(uids) if uids else total_count,
+        offset=effective_offset,
+        limit=limit,
+        uidvalidity=uidvalidity,
+        cursor_reset=cursor_reset or anchor_missing,
+    )
 
 
 def _has_folder_flag(flags: set[str], flag: str) -> bool:
@@ -691,9 +846,7 @@ def list_imap_messages(
     """List mailbox messages without mutating read/unread state."""
 
     host, port = _require_imap_config(imap_config)
-    folder = (folder or "INBOX").strip() or "INBOX"
-    limit = max(1, min(limit, 100))
-    offset = max(0, offset)
+    folder, limit, offset = _normalize_mailbox_page(folder=folder, limit=limit, offset=offset)
     if is_mock_imap_host(imap_config.host):
         records = [record for record in list_records(limit=500) if _mock_folder_matches(record, folder)]
         cursor_reset = False
@@ -736,6 +889,21 @@ def list_imap_messages(
     client = _open_imap(imap_config)
     try:
         total_count, uidvalidity = _select_readonly(client, folder)
+        if after_uid is None and expected_uidvalidity is None:
+            page_sequences = _paged_descending_sequences(total_count, offset=offset, limit=limit)
+            messages = _fetch_message_summaries_by_sequence(client, page_sequences, folder)
+            return ImapMailboxMessageListResult(
+                host=host,
+                port=port,
+                security=imap_config.security.value,
+                folder=folder,
+                messages=messages,
+                total_count=total_count,
+                offset=offset,
+                limit=limit,
+                uidvalidity=None,
+                cursor_reset=False,
+            )
         uids = _search_message_uids(client)
         cursor_reset = False
         effective_after_uid = after_uid
@@ -766,8 +934,12 @@ def list_imap_messages(
     finally:
         try:
             client.logout()
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            _log_imap_cleanup_failure("listing messages", cleanup_exc)
+
+
+def _normalize_mailbox_page(*, folder: str | None, limit: int, offset: int) -> tuple[str, int, int]:
+    return (folder or "INBOX").strip() or "INBOX", max(1, min(limit, 100)), max(0, offset)
 
 
 def _paged_descending_sequences(total_count: int, *, offset: int, limit: int) -> list[str]:
@@ -809,8 +981,8 @@ def get_imap_message(*, imap_config: ImapConfig, folder: str, uid: str) -> ImapM
     finally:
         try:
             client.logout()
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            _log_imap_cleanup_failure("reading message", cleanup_exc)
 
 
 def append_message_to_sent(
@@ -867,5 +1039,5 @@ def append_message_to_sent(
         if client is not None:
             try:
                 client.logout()
-            except Exception:
-                pass
+            except Exception as cleanup_exc:
+                _log_imap_cleanup_failure("appending sent message", cleanup_exc)

src/govoplan_mail/backend/sending/rate_limit.py

@@ -40,8 +40,8 @@ def wait_for_rate_limit(*, key: str, messages_per_minute: int, enabled: bool = T
     if not enabled or not _distributed_rate_limit_enabled():
         return RateLimitDecision(key=key, messages_per_minute=messages_per_minute, gap_seconds=gap, waited_seconds=0.0)
 
-    redis_key = f"multimailer:ratelimit:{key}:next_allowed"
-    lock_key = f"multimailer:ratelimit:{key}:lock"
+    redis_key = f"govoplan:ratelimit:{key}:next_allowed"
+    lock_key = f"govoplan:ratelimit:{key}:lock"
     waited = 0.0
 
     try:

src/govoplan_mail/backend/sending/smtp.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import copy
+import logging
 import smtplib
 import ssl
 from dataclasses import dataclass
@@ -15,6 +16,8 @@ from govoplan_mail.backend.dev.mock_mailbox import (
     record_smtp_delivery,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class SmtpConfigurationError(ValueError):
     """Raised when SMTP settings are incomplete or inconsistent."""
@@ -56,6 +59,10 @@ class SmtpSendResult:
         return len(self.envelope_recipients) - len(self.refused_recipients)
 
 
+def _log_smtp_cleanup_failure(action: str, exc: BaseException) -> None:
+    logger.debug("SMTP cleanup failed while %s: %s", action, exc, exc_info=True)
+
+
 def _require_smtp_config(config: SmtpConfig) -> tuple[str, int]:
     if not config.host:
         raise SmtpConfigurationError("SMTP host is required")
@@ -89,8 +96,8 @@ def _open_smtp(config: SmtpConfig) -> smtplib.SMTP:
         # on GC, but explicit cleanup is safer when the variable exists.
         try:
             smtp.quit()  # type: ignore[possibly-undefined]
-        except Exception:
-            pass
+        except Exception as cleanup_exc:
+            _log_smtp_cleanup_failure("opening connection", cleanup_exc)
         raise
 
 
@@ -134,11 +141,12 @@ def test_smtp_login(*, smtp_config: SmtpConfig) -> SmtpLoginTestResult:
     finally:
         try:
             smtp.quit()
-        except Exception:
+        except Exception as quit_exc:
+            _log_smtp_cleanup_failure("testing login quit", quit_exc)
             try:
                 smtp.close()
-            except Exception:
-                pass
+            except Exception as close_exc:
+                _log_smtp_cleanup_failure("testing login close", close_exc)
 
 
 def prepare_test_message(
@@ -161,12 +169,12 @@ def prepare_test_message(
             del test_message[header]
 
     # Replace potential previous marker headers if the user test-sends an EML twice.
-    for header in ["X-MultiMailer-Test-Send"]:
+    for header in ["X-GovOPlaN-Test-Send"]:
         if header in test_message:
             del test_message[header]
 
     test_message["To"] = formataddr((test_recipient_name or test_recipient, test_recipient))
-    test_message["X-MultiMailer-Test-Send"] = "true"
+    test_message["X-GovOPlaN-Test-Send"] = "true"
     return test_message
 
 
@@ -177,43 +185,97 @@ def _send_smtp_payload(
     envelope_from: str,
     envelope_recipients: list[str],
 ) -> SmtpSendResult:
+    host, port, recipients = _prepare_smtp_send(
+        smtp_config=smtp_config,
+        envelope_from=envelope_from,
+        envelope_recipients=envelope_recipients,
+    )
+
+    if is_mock_smtp_host(smtp_config.host):
+        _accepted, refused = _send_mock_smtp_payload(
+            message,
+            smtp_config=smtp_config,
+            envelope_from=envelope_from,
+            envelope_recipients=recipients,
+        )
+        return _smtp_send_result(
+            smtp_config=smtp_config,
+            host=host,
+            port=port,
+            envelope_from=envelope_from,
+            envelope_recipients=recipients,
+            refused=refused,
+        )
+
+    refused = _send_network_smtp_payload(
+        message,
+        smtp_config=smtp_config,
+        envelope_from=envelope_from,
+        envelope_recipients=recipients,
+    )
+    return _smtp_send_result(
+        smtp_config=smtp_config,
+        host=host,
+        port=port,
+        envelope_from=envelope_from,
+        envelope_recipients=recipients,
+        refused=refused,
+    )
+
+
+def _prepare_smtp_send(
+    *,
+    smtp_config: SmtpConfig,
+    envelope_from: str,
+    envelope_recipients: list[str],
+) -> tuple[str, int, list[str]]:
     host, port = _require_smtp_config(smtp_config)
     if not envelope_from:
         raise SmtpConfigurationError("SMTP envelope sender is required")
-    if not envelope_recipients:
+    recipients = [recipient for recipient in envelope_recipients if recipient]
+    if not recipients:
         raise SmtpConfigurationError("at least one SMTP envelope recipient is required")
+    return host, port, recipients
 
-    if is_mock_smtp_host(smtp_config.host):
-        if consume_fail_next_smtp():
-            raise SmtpSendError("Mock SMTP configured to fail the next send")
-        failures = get_failures()
-        reject_text = str(failures.get("smtp_reject_recipients_containing") or "").strip().lower()
-        refused: dict[str, tuple[int, bytes]] = {}
-        accepted = list(envelope_recipients)
-        if reject_text:
-            refused = {
-                recipient: (550, b"mock recipient rejected")
-                for recipient in envelope_recipients
-                if reject_text in recipient.lower()
-            }
-            accepted = [recipient for recipient in envelope_recipients if recipient not in refused]
-            if not accepted:
-                raise SmtpSendError(f"all mock SMTP recipients were refused: {_decode_refused(refused)}")
-        record_smtp_delivery(
-            message,
-            envelope_from=envelope_from,
-            envelope_recipients=accepted,
-            smtp_host=smtp_config.host,
-        )
-        return SmtpSendResult(
-            host=host,
-            port=port,
-            security=smtp_config.security.value,
-            envelope_from=envelope_from,
-            envelope_recipients=list(envelope_recipients),
-            refused_recipients=_decode_refused(refused),
-        )
 
+def _send_mock_smtp_payload(
+    message: EmailMessage | bytes,
+    *,
+    smtp_config: SmtpConfig,
+    envelope_from: str,
+    envelope_recipients: list[str],
+) -> tuple[list[str], dict[str, tuple[int, bytes]]]:
+    if consume_fail_next_smtp():
+        raise SmtpSendError("Mock SMTP configured to fail the next send")
+    failures = get_failures()
+    reject_text = str(failures.get("smtp_reject_recipients_containing") or "").strip().lower()
+    refused: dict[str, tuple[int, bytes]] = {}
+    accepted = list(envelope_recipients)
+    if reject_text:
+        refused = {
+            recipient: (550, b"mock recipient rejected")
+            for recipient in envelope_recipients
+            if reject_text in recipient.lower()
+        }
+        accepted = [recipient for recipient in envelope_recipients if recipient not in refused]
+        if not accepted:
+            raise SmtpSendError(f"all mock SMTP recipients were refused: {_decode_refused(refused)}")
+    record_smtp_delivery(
+        message,
+        envelope_from=envelope_from,
+        envelope_recipients=accepted,
+        smtp_host=smtp_config.host,
+    )
+    return accepted, refused
+
+
+def _send_network_smtp_payload(
+    message: EmailMessage | bytes,
+    *,
+    smtp_config: SmtpConfig,
+    envelope_from: str,
+    envelope_recipients: list[str],
+) -> dict[str, tuple[int, bytes]]:
     try:
         smtp = _open_smtp(smtp_config)
     except smtplib.SMTPAuthenticationError as exc:
@@ -266,12 +328,24 @@ def _send_smtp_payload(
     finally:
         try:
             smtp.quit()
-        except Exception:
+        except Exception as quit_exc:
+            _log_smtp_cleanup_failure("sending message quit", quit_exc)
             try:
                 smtp.close()
-            except Exception:
-                pass
+            except Exception as close_exc:
+                _log_smtp_cleanup_failure("sending message close", close_exc)
+    return refused
 
+
+def _smtp_send_result(
+    *,
+    smtp_config: SmtpConfig,
+    host: str,
+    port: int,
+    envelope_from: str,
+    envelope_recipients: list[str],
+    refused: dict[str, tuple[int, bytes]],
+) -> SmtpSendResult:
     return SmtpSendResult(
         host=host,
         port=port,