intermittent commit

2026-07-14 13:22:11 +02:00
parent 4865de5007
commit b0337b2d26
28 changed files with 2264 additions and 490 deletions
--- a/src/govoplan_mail/backend/mailbox_index.py
+++ b/src/govoplan_mail/backend/mailbox_index.py
@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from threading import Lock
+
+from sqlalchemy import func
+from sqlalchemy.orm import Session
+
+from govoplan_core.security.time import ensure_aware_utc, utc_now
+from govoplan_mail.backend.db.models import MailMailboxFolderIndex, MailMailboxMessageIndex
+from govoplan_mail.backend.sending.imap import ImapFolderListResult, ImapMailboxInfo, ImapMailboxMessageListResult, ImapMailboxMessageSummary
+
+MAILBOX_INDEX_TTL_SECONDS = 30
+
+_refresh_lock = Lock()
+_refreshing_keys: set[tuple[str, str, str]] = set()
+
+
+@dataclass(frozen=True, slots=True)
+class CachedFolderList:
+    folders: list[ImapMailboxInfo]
+    indexed_at: datetime | None
+    stale: bool
+
+
+@dataclass(frozen=True, slots=True)
+class CachedMessagePage:
+    folder: str
+    messages: list[ImapMailboxMessageSummary]
+    total_count: int
+    offset: int
+    limit: int
+    uidvalidity: str | None
+    indexed_at: datetime | None
+    stale: bool
+
+
+def begin_mailbox_refresh(tenant_id: str, profile_id: str, folder: str) -> bool:
+    key = (tenant_id, profile_id, folder)
+    with _refresh_lock:
+        if key in _refreshing_keys:
+            return False
+        _refreshing_keys.add(key)
+        return True
+
+
+def finish_mailbox_refresh(tenant_id: str, profile_id: str, folder: str) -> None:
+    with _refresh_lock:
+        _refreshing_keys.discard((tenant_id, profile_id, folder))
+
+
+def cache_mailbox_folders(
+    session: Session,
+    *,
+    tenant_id: str,
+    profile_id: str,
+    result: ImapFolderListResult,
+    indexed_at: datetime | None = None,
+) -> None:
+    indexed_at = indexed_at or utc_now()
+    existing = {
+        row.folder: row
+        for row in session.query(MailMailboxFolderIndex)
+        .filter(MailMailboxFolderIndex.tenant_id == tenant_id, MailMailboxFolderIndex.profile_id == profile_id)
+        .all()
+    }
+    for folder in result.folders:
+        row = existing.get(folder.name)
+        if row is None:
+            row = MailMailboxFolderIndex(tenant_id=tenant_id, profile_id=profile_id, folder=folder.name)
+        row.flags = list(folder.flags or [])
+        row.message_count = folder.message_count
+        row.unseen_count = folder.unseen_count
+        row.indexed_at = indexed_at
+        session.add(row)
+
+
+def cache_mailbox_messages(
+    session: Session,
+    *,
+    tenant_id: str,
+    profile_id: str,
+    result: ImapMailboxMessageListResult,
+    indexed_at: datetime | None = None,
+) -> None:
+    indexed_at = indexed_at or utc_now()
+    session.flush()
+    folder_row = (
+        session.query(MailMailboxFolderIndex)
+        .filter(
+            MailMailboxFolderIndex.tenant_id == tenant_id,
+            MailMailboxFolderIndex.profile_id == profile_id,
+            MailMailboxFolderIndex.folder == result.folder,
+        )
+        .one_or_none()
+    )
+    if folder_row is None:
+        folder_row = MailMailboxFolderIndex(tenant_id=tenant_id, profile_id=profile_id, folder=result.folder)
+    folder_row.message_count = result.total_count
+    folder_row.uidvalidity = result.uidvalidity
+    folder_row.message_indexed_at = indexed_at
+    session.add(folder_row)
+
+    uids = [message.uid for message in result.messages]
+    if result.total_count <= 0:
+        (
+            session.query(MailMailboxMessageIndex)
+            .filter(
+                MailMailboxMessageIndex.tenant_id == tenant_id,
+                MailMailboxMessageIndex.profile_id == profile_id,
+                MailMailboxMessageIndex.folder == result.folder,
+            )
+            .delete(synchronize_session=False)
+        )
+        return
+    window_count = max(0, min(result.limit, result.total_count - result.offset))
+    if window_count:
+        stale_query = session.query(MailMailboxMessageIndex).filter(
+            MailMailboxMessageIndex.tenant_id == tenant_id,
+            MailMailboxMessageIndex.profile_id == profile_id,
+            MailMailboxMessageIndex.folder == result.folder,
+            MailMailboxMessageIndex.sort_position >= result.offset,
+            MailMailboxMessageIndex.sort_position < result.offset + window_count,
+        )
+        if uids:
+            stale_query = stale_query.filter(MailMailboxMessageIndex.uid.notin_(uids))
+        for row in stale_query.all():
+            session.delete(row)
+    existing = {}
+    if uids:
+        existing = {
+            row.uid: row
+            for row in session.query(MailMailboxMessageIndex)
+            .filter(
+                MailMailboxMessageIndex.tenant_id == tenant_id,
+                MailMailboxMessageIndex.profile_id == profile_id,
+                MailMailboxMessageIndex.folder == result.folder,
+                MailMailboxMessageIndex.uid.in_(uids),
+            )
+            .all()
+        }
+    for index, message in enumerate(result.messages):
+        row = existing.get(message.uid)
+        if row is None:
+            row = MailMailboxMessageIndex(
+                tenant_id=tenant_id,
+                profile_id=profile_id,
+                folder=result.folder,
+                uid=message.uid,
+            )
+        row.uid_int = _uid_int(message.uid)
+        row.sort_position = result.offset + index
+        row.subject = message.subject
+        row.from_header = message.from_header
+        row.to_header = message.to_header
+        row.cc_header = message.cc_header
+        row.date = message.date
+        row.message_id = message.message_id
+        row.flags = list(message.flags or [])
+        row.size_bytes = message.size_bytes
+        row.body_preview = message.body_preview
+        row.attachment_count = message.attachment_count
+        row.indexed_at = indexed_at
+        session.add(row)
+
+
+def cached_mailbox_folders(
+    session: Session,
+    *,
+    tenant_id: str,
+    profile_id: str,
+    max_age_seconds: int = MAILBOX_INDEX_TTL_SECONDS,
+) -> CachedFolderList | None:
+    rows = (
+        session.query(MailMailboxFolderIndex)
+        .filter(
+            MailMailboxFolderIndex.tenant_id == tenant_id,
+            MailMailboxFolderIndex.profile_id == profile_id,
+            MailMailboxFolderIndex.indexed_at.isnot(None),
+        )
+        .order_by(MailMailboxFolderIndex.folder.asc())
+        .all()
+    )
+    if not rows:
+        return None
+    indexed_at = max((row.indexed_at for row in rows if row.indexed_at), default=None)
+    return CachedFolderList(
+        folders=[
+            ImapMailboxInfo(name=row.folder, flags=list(row.flags or []), message_count=row.message_count, unseen_count=row.unseen_count)
+            for row in rows
+        ],
+        indexed_at=indexed_at,
+        stale=_is_stale(indexed_at, max_age_seconds=max_age_seconds),
+    )
+
+
+def cached_mailbox_message_page(
+    session: Session,
+    *,
+    tenant_id: str,
+    profile_id: str,
+    folder: str,
+    limit: int,
+    offset: int,
+    max_age_seconds: int = MAILBOX_INDEX_TTL_SECONDS,
+) -> CachedMessagePage | None:
+    folder_row = (
+        session.query(MailMailboxFolderIndex)
+        .filter(
+            MailMailboxFolderIndex.tenant_id == tenant_id,
+            MailMailboxFolderIndex.profile_id == profile_id,
+            MailMailboxFolderIndex.folder == folder,
+        )
+        .one_or_none()
+    )
+    if folder_row is None or folder_row.message_indexed_at is None:
+        return None
+    total_count = folder_row.message_count
+    if total_count is None:
+        total_count = int(
+            session.query(func.count(MailMailboxMessageIndex.id))
+            .filter(
+                MailMailboxMessageIndex.tenant_id == tenant_id,
+                MailMailboxMessageIndex.profile_id == profile_id,
+                MailMailboxMessageIndex.folder == folder,
+            )
+            .scalar()
+            or 0
+        )
+    rows = (
+        session.query(MailMailboxMessageIndex)
+        .filter(
+            MailMailboxMessageIndex.tenant_id == tenant_id,
+            MailMailboxMessageIndex.profile_id == profile_id,
+            MailMailboxMessageIndex.folder == folder,
+        )
+        .order_by(MailMailboxMessageIndex.sort_position.asc(), MailMailboxMessageIndex.uid_int.desc(), MailMailboxMessageIndex.uid.desc())
+        .offset(offset)
+        .limit(limit)
+        .all()
+    )
+    expected_count = max(0, min(limit, total_count - offset))
+    if len(rows) < expected_count:
+        return None
+    indexed_at = min((row.indexed_at for row in rows if row.indexed_at), default=folder_row.message_indexed_at)
+    return CachedMessagePage(
+        folder=folder,
+        messages=[_message_from_index(row) for row in rows],
+        total_count=total_count,
+        offset=offset,
+        limit=limit,
+        uidvalidity=folder_row.uidvalidity,
+        indexed_at=indexed_at,
+        stale=_is_stale(indexed_at, max_age_seconds=max_age_seconds),
+    )
+
+
+def _message_from_index(row: MailMailboxMessageIndex) -> ImapMailboxMessageSummary:
+    return ImapMailboxMessageSummary(
+        uid=row.uid,
+        folder=row.folder,
+        subject=row.subject,
+        from_header=row.from_header,
+        to_header=row.to_header,
+        cc_header=row.cc_header,
+        date=row.date,
+        message_id=row.message_id,
+        flags=list(row.flags or []),
+        size_bytes=row.size_bytes,
+        body_preview=row.body_preview,
+        attachment_count=row.attachment_count,
+    )
+
+
+def _uid_int(uid: str) -> int:
+    try:
+        return int(str(uid))
+    except ValueError:
+        return 0
+
+
+def _is_stale(indexed_at: datetime | None, *, max_age_seconds: int) -> bool:
+    aware = ensure_aware_utc(indexed_at)
+    if aware is None:
+        return True
+    return (utc_now() - aware).total_seconds() > max_age_seconds