GovOPlaN/govoplan-mail
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

intermittent commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-14 13:22:11 +02:00
parent 4865de5007
commit b0337b2d26
28 changed files with 2264 additions and 490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/govoplan_mail/backend/config.py
View File

@@ -8,6 +8,7 @@ from govoplan_core.mail.config import (
StrictModel,
TransportCredentials,
TransportSecurity,
normalize_split_transport_credentials,
)
__all__ = [
@@ -18,4 +19,5 @@ __all__ = [
"StrictModel",
"TransportCredentials",
"TransportSecurity",
"normalize_split_transport_credentials",
]

50
src/govoplan_mail/backend/db/models.py
View File

@@ -3,7 +3,9 @@ from __future__ import annotations
import uuid
from typing import Any
from sqlalchemy import Boolean, ForeignKey, Index, JSON, String, Text, UniqueConstraint
from datetime import datetime
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from govoplan_core.db.base import Base, TimestampMixin
@@ -50,3 +52,49 @@ class MailProfilePolicy(Base, TimestampMixin):
scope_type: Mapped[str] = mapped_column(String(20), nullable=False, index=True)
scope_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
policy: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False)
class MailMailboxFolderIndex(Base, TimestampMixin):
__tablename__ = "mail_mailbox_folder_index"
__table_args__ = (
UniqueConstraint("profile_id", "folder", name="uq_mail_mailbox_folder_index_profile_folder"),
Index("ix_mail_mailbox_folder_index_tenant_profile", "tenant_id", "profile_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(ForeignKey("mail_server_profiles.id", ondelete="CASCADE"), nullable=False, index=True)
folder: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
flags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
message_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
unseen_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
uidvalidity: Mapped[str | None] = mapped_column(String(255), nullable=True)
indexed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
message_indexed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
class MailMailboxMessageIndex(Base, TimestampMixin):
__tablename__ = "mail_mailbox_message_index"
__table_args__ = (
UniqueConstraint("profile_id", "folder", "uid", name="uq_mail_mailbox_message_index_profile_folder_uid"),
Index("ix_mail_mailbox_message_index_page", "tenant_id", "profile_id", "folder", "sort_position"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
profile_id: Mapped[str] = mapped_column(ForeignKey("mail_server_profiles.id", ondelete="CASCADE"), nullable=False, index=True)
folder: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
uid: Mapped[str] = mapped_column(String(255), nullable=False)
uid_int: Mapped[int] = mapped_column(BigInteger, default=0, nullable=False, index=True)
sort_position: Mapped[int] = mapped_column(BigInteger, default=0, nullable=False, index=True)
subject: Mapped[str | None] = mapped_column(Text, nullable=True)
from_header: Mapped[str | None] = mapped_column(Text, nullable=True)
to_header: Mapped[str | None] = mapped_column(Text, nullable=True)
cc_header: Mapped[str | None] = mapped_column(Text, nullable=True)
date: Mapped[str | None] = mapped_column(String(255), nullable=True)
message_id: Mapped[str | None] = mapped_column(Text, nullable=True)
flags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
body_preview: Mapped[str | None] = mapped_column(Text, nullable=True)
attachment_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
indexed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)

169
src/govoplan_mail/backend/mail_profiles.py
View File

@@ -496,15 +496,7 @@ def _merge_policy(policy: EffectiveMailProfilePolicy, data: dict[str, Any], *, s
normalized = normalize_mail_profile_policy(data)
policy.source_policies.append(_mail_policy_source_step(source, source_id, label or source.capitalize(), normalized, baseline=baseline))
lower_limits = dict(policy.allow_lower_level_limits)
allowed_ids = _meaningful_allow_patterns(normalized.get("allowed_profile_ids") or [])
if allowed_ids and lower_limits.get("allowed_profile_ids", True):
policy.allowed_profile_id_sets.append(set(allowed_ids))
if normalized.get("allow_user_profiles") is False and lower_limits.get("allow_user_profiles", True):
policy.allow_user_profiles = False
if normalized.get("allow_group_profiles") is False and lower_limits.get("allow_group_profiles", True):
policy.allow_group_profiles = False
if normalized.get("allow_campaign_profiles") is False and lower_limits.get("allow_campaign_profiles", True):
policy.allow_campaign_profiles = False
_merge_profile_definition_policy(policy, normalized, lower_limits)
_merge_credential_policy(
policy.smtp_credentials,
normalized.get("smtp_credentials") or {},
@@ -519,19 +511,43 @@ def _merge_policy(policy: EffectiveMailProfilePolicy, data: dict[str, Any], *, s
lower_limits=lower_limits,
inherit_limit_key="imap_credentials.inherit",
)
_merge_pattern_policy(policy, normalized, lower_limits)
_merge_lower_level_limits(policy, normalized, lower_limits)
def _policy_limit_enabled(lower_limits: dict[str, bool], key: str) -> bool:
return lower_limits.get(key, True)
def _merge_profile_definition_policy(policy: EffectiveMailProfilePolicy, normalized: dict[str, Any], lower_limits: dict[str, bool]) -> None:
allowed_ids = _meaningful_allow_patterns(normalized.get("allowed_profile_ids") or [])
if allowed_ids and _policy_limit_enabled(lower_limits, "allowed_profile_ids"):
policy.allowed_profile_id_sets.append(set(allowed_ids))
if normalized.get("allow_user_profiles") is False and _policy_limit_enabled(lower_limits, "allow_user_profiles"):
policy.allow_user_profiles = False
if normalized.get("allow_group_profiles") is False and _policy_limit_enabled(lower_limits, "allow_group_profiles"):
policy.allow_group_profiles = False
if normalized.get("allow_campaign_profiles") is False and _policy_limit_enabled(lower_limits, "allow_campaign_profiles"):
policy.allow_campaign_profiles = False
def _merge_pattern_policy(policy: EffectiveMailProfilePolicy, normalized: dict[str, Any], lower_limits: dict[str, bool]) -> None:
whitelist = normalized.get("whitelist") or {}
blacklist = normalized.get("blacklist") or {}
for key in PROFILE_PATTERN_KEYS:
allow_patterns = _meaningful_allow_patterns(whitelist.get(key, []))
if allow_patterns and lower_limits.get(f"whitelist.{key}", True):
if allow_patterns and _policy_limit_enabled(lower_limits, f"whitelist.{key}"):
policy.whitelist_groups.setdefault(key, []).append(allow_patterns)
deny_patterns = _clean_string_list(blacklist.get(key, []))
if deny_patterns and lower_limits.get(f"blacklist.{key}", True):
if deny_patterns and _policy_limit_enabled(lower_limits, f"blacklist.{key}"):
policy.blacklist_patterns.setdefault(key, []).extend(deny_patterns)
def _merge_lower_level_limits(policy: EffectiveMailProfilePolicy, normalized: dict[str, Any], lower_limits: dict[str, bool]) -> None:
local_lower_limits = normalized.get("allow_lower_level_limits") or {}
if local_lower_limits:
policy.allow_lower_level_limits = {
key: lower_limits.get(key, True) and local_lower_limits.get(key, lower_limits.get(key, True))
key: _policy_limit_enabled(lower_limits, key) and local_lower_limits.get(key, _policy_limit_enabled(lower_limits, key))
for key in MAIL_POLICY_LIMIT_KEYS
}
@@ -1187,49 +1203,81 @@ def update_mail_server_profile(
if is_active is not None:
profile.is_active = is_active
next_smtp = smtp or SmtpConfig.model_validate({**(profile.smtp_config or {}), "username": _profile_username(profile, "smtp"), "password": decrypt_secret(profile.smtp_password_encrypted)})
if clear_imap:
next_imap = None
elif imap is not None:
next_imap = imap
elif profile.imap_config:
next_imap = ImapConfig.model_validate({**(profile.imap_config or {}), "username": _profile_username(profile, "imap"), "password": decrypt_secret(profile.imap_password_encrypted)})
else:
next_imap = None
next_smtp, next_imap = _next_profile_transport_state(profile, smtp=smtp, imap=imap, clear_imap=clear_imap)
_assert_profile_transport_allowed(session, tenant_id=tenant_id, profile=profile, smtp=next_smtp, imap=next_imap)
_apply_profile_transport_update(profile, smtp=smtp, imap=imap, clear_imap=clear_imap)
profile.updated_by_user_id = user_id
session.add(profile)
session.flush()
return profile
def _next_profile_transport_state(
profile: MailServerProfile,
*,
smtp: SmtpConfig | None,
imap: ImapConfig | None,
clear_imap: bool,
) -> tuple[SmtpConfig, ImapConfig | None]:
next_smtp = smtp or smtp_config_from_profile(profile)
if clear_imap:
return next_smtp, None
if imap is not None:
return next_smtp, imap
return next_smtp, imap_config_from_profile(profile)
def _assert_profile_transport_allowed(
session: Session,
*,
tenant_id: str,
profile: MailServerProfile,
smtp: SmtpConfig,
imap: ImapConfig | None,
) -> None:
scope_type = _profile_scope_type(profile)
scope_id = _profile_scope_id(profile)
if scope_type == "campaign":
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=next_smtp, imap=next_imap, campaign_id=scope_id)
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=smtp, imap=imap, campaign_id=scope_id)
elif scope_type == "user":
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=next_smtp, imap=next_imap, owner_user_id=scope_id)
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=smtp, imap=imap, owner_user_id=scope_id)
elif scope_type == "group":
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=next_smtp, imap=next_imap, owner_group_id=scope_id)
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=smtp, imap=imap, owner_group_id=scope_id)
elif scope_type == "tenant":
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=next_smtp, imap=next_imap)
assert_mail_policy_allows_transport(session, tenant_id=tenant_id, smtp=smtp, imap=imap)
def _apply_profile_transport_update(
profile: MailServerProfile,
*,
smtp: SmtpConfig | None,
imap: ImapConfig | None,
clear_imap: bool,
) -> None:
if smtp is not None:
smtp_payload, smtp_username, smtp_password, username_supplied, password_supplied = _transport_payload(smtp)
profile.smtp_config = smtp_payload
if username_supplied:
profile.smtp_username = smtp_username
if password_supplied:
profile.smtp_password_encrypted = encrypt_secret(smtp_password)
_apply_profile_transport_payload(profile, "smtp", smtp)
if clear_imap:
profile.imap_config = None
profile.imap_username = None
profile.imap_password_encrypted = None
elif imap is not None:
imap_payload, imap_username, imap_password, username_supplied, password_supplied = _transport_payload(imap)
profile.imap_config = imap_payload
_apply_profile_transport_payload(profile, "imap", imap)
def _apply_profile_transport_payload(profile: MailServerProfile, protocol: str, config: SmtpConfig | ImapConfig) -> None:
payload, username, password, username_supplied, password_supplied = _transport_payload(config)
if protocol == "smtp":
profile.smtp_config = payload
if username_supplied:
profile.imap_username = imap_username
profile.smtp_username = username
if password_supplied:
profile.imap_password_encrypted = encrypt_secret(imap_password)
profile.updated_by_user_id = user_id
session.add(profile)
session.flush()
return profile
profile.smtp_password_encrypted = encrypt_secret(password)
return
profile.imap_config = payload
if username_supplied:
profile.imap_username = username
if password_supplied:
profile.imap_password_encrypted = encrypt_secret(password)
def _profile_username(profile: MailServerProfile, protocol: str) -> str | None:
@@ -1346,29 +1394,50 @@ def get_mail_profile_policy(
def _validate_policy_against_parent(parent: EffectiveMailProfilePolicy, normalized: dict[str, Any]) -> None:
parent_limits = parent.allow_lower_level_limits
violations = _policy_parent_lock_violations(parent.allow_lower_level_limits, normalized)
if violations:
raise MailProfileError(_policy_parent_lock_message(violations[0]))
def _policy_parent_lock_violations(parent_limits: dict[str, bool], normalized: dict[str, Any]) -> list[str]:
violations: list[str] = []
allowed_ids = _meaningful_allow_patterns(normalized.get("allowed_profile_ids") or [])
if allowed_ids and not parent_limits.get("allowed_profile_ids", True):
raise MailProfileError("Mail profile allow-list is locked by an ancestor policy")
if allowed_ids and not _policy_limit_enabled(parent_limits, "allowed_profile_ids"):
violations.append("allowed_profile_ids")
for key in ("allow_user_profiles", "allow_group_profiles", "allow_campaign_profiles"):
if isinstance(normalized.get(key), bool) and not parent_limits.get(key, True):
raise MailProfileError(f"{key} is locked by an ancestor policy")
if isinstance(normalized.get(key), bool) and not _policy_limit_enabled(parent_limits, key):
violations.append(key)
for kind in ("whitelist", "blacklist"):
rules = normalized.get(kind) or {}
if isinstance(rules, dict):
for key in PROFILE_PATTERN_KEYS:
if _clean_string_list(rules.get(key, [])) and not parent_limits.get(f"{kind}.{key}", True):
raise MailProfileError(f"{kind}.{key} is locked by an ancestor policy")
limit_key = f"{kind}.{key}"
if _clean_string_list(rules.get(key, [])) and not _policy_limit_enabled(parent_limits, limit_key):
violations.append(limit_key)
for protocol in ("smtp", "imap"):
local = normalized.get(f"{protocol}_credentials") or {}
local_inherit = local.get("inherit")
if isinstance(local_inherit, bool) and not parent_limits.get(f"{protocol}_credentials.inherit", True):
raise MailProfileError(f"{protocol.upper()} credential inheritance is locked by an ancestor policy")
limit_key = f"{protocol}_credentials.inherit"
if isinstance(local_inherit, bool) and not _policy_limit_enabled(parent_limits, limit_key):
violations.append(limit_key)
local_lower_limits = normalized.get("allow_lower_level_limits") or {}
if isinstance(local_lower_limits, dict):
for key, allowed in local_lower_limits.items():
if allowed is True and not parent_limits.get(key, True):
raise MailProfileError(f"{key} lower-level limiting cannot be re-enabled below a locked ancestor policy")
if allowed is True and not _policy_limit_enabled(parent_limits, key):
violations.append(f"allow_lower_level_limits.{key}")
return violations
def _policy_parent_lock_message(field: str) -> str:
if field == "allowed_profile_ids":
return "Mail profile allow-list is locked by an ancestor policy"
if field.startswith("allow_lower_level_limits."):
key = field.removeprefix("allow_lower_level_limits.")
return f"{key} lower-level limiting cannot be re-enabled below a locked ancestor policy"
if field.endswith("_credentials.inherit"):
protocol = field.split("_", 1)[0].upper()
return f"{protocol} credential inheritance is locked by an ancestor policy"
return f"{field} is locked by an ancestor policy"
def set_mail_profile_policy(

287
src/govoplan_mail/backend/mailbox_index.py Normal file
View File

@@ -0,0 +1,287 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
from sqlalchemy import func
from sqlalchemy.orm import Session
from govoplan_core.security.time import ensure_aware_utc, utc_now
from govoplan_mail.backend.db.models import MailMailboxFolderIndex, MailMailboxMessageIndex
from govoplan_mail.backend.sending.imap import ImapFolderListResult, ImapMailboxInfo, ImapMailboxMessageListResult, ImapMailboxMessageSummary
MAILBOX_INDEX_TTL_SECONDS = 30
_refresh_lock = Lock()
_refreshing_keys: set[tuple[str, str, str]] = set()
@dataclass(frozen=True, slots=True)
class CachedFolderList:
folders: list[ImapMailboxInfo]
indexed_at: datetime | None
stale: bool
@dataclass(frozen=True, slots=True)
class CachedMessagePage:
folder: str
messages: list[ImapMailboxMessageSummary]
total_count: int
offset: int
limit: int
uidvalidity: str | None
indexed_at: datetime | None
stale: bool
def begin_mailbox_refresh(tenant_id: str, profile_id: str, folder: str) -> bool:
key = (tenant_id, profile_id, folder)
with _refresh_lock:
if key in _refreshing_keys:
return False
_refreshing_keys.add(key)
return True
def finish_mailbox_refresh(tenant_id: str, profile_id: str, folder: str) -> None:
with _refresh_lock:
_refreshing_keys.discard((tenant_id, profile_id, folder))
def cache_mailbox_folders(
session: Session,
*,
tenant_id: str,
profile_id: str,
result: ImapFolderListResult,
indexed_at: datetime | None = None,
) -> None:
indexed_at = indexed_at or utc_now()
existing = {
row.folder: row
for row in session.query(MailMailboxFolderIndex)
.filter(MailMailboxFolderIndex.tenant_id == tenant_id, MailMailboxFolderIndex.profile_id == profile_id)
.all()
}
for folder in result.folders:
row = existing.get(folder.name)
if row is None:
row = MailMailboxFolderIndex(tenant_id=tenant_id, profile_id=profile_id, folder=folder.name)
row.flags = list(folder.flags or [])
row.message_count = folder.message_count
row.unseen_count = folder.unseen_count
row.indexed_at = indexed_at
session.add(row)
def cache_mailbox_messages(
session: Session,
*,
tenant_id: str,
profile_id: str,
result: ImapMailboxMessageListResult,
indexed_at: datetime | None = None,
) -> None:
indexed_at = indexed_at or utc_now()
session.flush()
folder_row = (
session.query(MailMailboxFolderIndex)
.filter(
MailMailboxFolderIndex.tenant_id == tenant_id,
MailMailboxFolderIndex.profile_id == profile_id,
MailMailboxFolderIndex.folder == result.folder,
)
.one_or_none()
)
if folder_row is None:
folder_row = MailMailboxFolderIndex(tenant_id=tenant_id, profile_id=profile_id, folder=result.folder)
folder_row.message_count = result.total_count
folder_row.uidvalidity = result.uidvalidity
folder_row.message_indexed_at = indexed_at
session.add(folder_row)
uids = [message.uid for message in result.messages]
if result.total_count <= 0:
(
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
)
.delete(synchronize_session=False)
)
return
window_count = max(0, min(result.limit, result.total_count - result.offset))
if window_count:
stale_query = session.query(MailMailboxMessageIndex).filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
MailMailboxMessageIndex.sort_position >= result.offset,
MailMailboxMessageIndex.sort_position < result.offset + window_count,
)
if uids:
stale_query = stale_query.filter(MailMailboxMessageIndex.uid.notin_(uids))
for row in stale_query.all():
session.delete(row)
existing = {}
if uids:
existing = {
row.uid: row
for row in session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == result.folder,
MailMailboxMessageIndex.uid.in_(uids),
)
.all()
}
for index, message in enumerate(result.messages):
row = existing.get(message.uid)
if row is None:
row = MailMailboxMessageIndex(
tenant_id=tenant_id,
profile_id=profile_id,
folder=result.folder,
uid=message.uid,
)
row.uid_int = _uid_int(message.uid)
row.sort_position = result.offset + index
row.subject = message.subject
row.from_header = message.from_header
row.to_header = message.to_header
row.cc_header = message.cc_header
row.date = message.date
row.message_id = message.message_id
row.flags = list(message.flags or [])
row.size_bytes = message.size_bytes
row.body_preview = message.body_preview
row.attachment_count = message.attachment_count
row.indexed_at = indexed_at
session.add(row)
def cached_mailbox_folders(
session: Session,
*,
tenant_id: str,
profile_id: str,
max_age_seconds: int = MAILBOX_INDEX_TTL_SECONDS,
) -> CachedFolderList | None:
rows = (
session.query(MailMailboxFolderIndex)
.filter(
MailMailboxFolderIndex.tenant_id == tenant_id,
MailMailboxFolderIndex.profile_id == profile_id,
MailMailboxFolderIndex.indexed_at.isnot(None),
)
.order_by(MailMailboxFolderIndex.folder.asc())
.all()
)
if not rows:
return None
indexed_at = max((row.indexed_at for row in rows if row.indexed_at), default=None)
return CachedFolderList(
folders=[
ImapMailboxInfo(name=row.folder, flags=list(row.flags or []), message_count=row.message_count, unseen_count=row.unseen_count)
for row in rows
],
indexed_at=indexed_at,
stale=_is_stale(indexed_at, max_age_seconds=max_age_seconds),
)
def cached_mailbox_message_page(
session: Session,
*,
tenant_id: str,
profile_id: str,
folder: str,
limit: int,
offset: int,
max_age_seconds: int = MAILBOX_INDEX_TTL_SECONDS,
) -> CachedMessagePage | None:
folder_row = (
session.query(MailMailboxFolderIndex)
.filter(
MailMailboxFolderIndex.tenant_id == tenant_id,
MailMailboxFolderIndex.profile_id == profile_id,
MailMailboxFolderIndex.folder == folder,
)
.one_or_none()
)
if folder_row is None or folder_row.message_indexed_at is None:
return None
total_count = folder_row.message_count
if total_count is None:
total_count = int(
session.query(func.count(MailMailboxMessageIndex.id))
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == folder,
)
.scalar()
or 0
)
rows = (
session.query(MailMailboxMessageIndex)
.filter(
MailMailboxMessageIndex.tenant_id == tenant_id,
MailMailboxMessageIndex.profile_id == profile_id,
MailMailboxMessageIndex.folder == folder,
)
.order_by(MailMailboxMessageIndex.sort_position.asc(), MailMailboxMessageIndex.uid_int.desc(), MailMailboxMessageIndex.uid.desc())
.offset(offset)
.limit(limit)
.all()
)
expected_count = max(0, min(limit, total_count - offset))
if len(rows) < expected_count:
return None
indexed_at = min((row.indexed_at for row in rows if row.indexed_at), default=folder_row.message_indexed_at)
return CachedMessagePage(
folder=folder,
messages=[_message_from_index(row) for row in rows],
total_count=total_count,
offset=offset,
limit=limit,
uidvalidity=folder_row.uidvalidity,
indexed_at=indexed_at,
stale=_is_stale(indexed_at, max_age_seconds=max_age_seconds),
)
def _message_from_index(row: MailMailboxMessageIndex) -> ImapMailboxMessageSummary:
return ImapMailboxMessageSummary(
uid=row.uid,
folder=row.folder,
subject=row.subject,
from_header=row.from_header,
to_header=row.to_header,
cc_header=row.cc_header,
date=row.date,
message_id=row.message_id,
flags=list(row.flags or []),
size_bytes=row.size_bytes,
body_preview=row.body_preview,
attachment_count=row.attachment_count,
)
def _uid_int(uid: str) -> int:
try:
return int(str(uid))
except ValueError:
return 0
def _is_stale(indexed_at: datetime | None, *, max_age_seconds: int) -> bool:
aware = ensure_aware_utc(indexed_at)
if aware is None:
return True
return (utc_now() - aware).total_seconds() > max_age_seconds

14
src/govoplan_mail/backend/manifest.py
View File

@@ -72,7 +72,7 @@ ROLE_TEMPLATES = (
def _mail_router(context: ModuleContext):
from govoplan_mail.backend.runtime import configure_runtime
configure_runtime(settings=context.settings)
configure_runtime(registry=context.registry, settings=context.settings)
from fastapi import APIRouter
from govoplan_mail.backend.router import router
@@ -91,7 +91,7 @@ manifest = ModuleManifest(
name="Mail",
version="0.1.8",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("campaigns",),
optional_dependencies=("campaigns", "addresses"),
provides_interfaces=(
ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.1.6"),
),
@@ -102,6 +102,12 @@ manifest = ModuleManifest(
version_max_exclusive="0.2.0",
optional=True,
),
ModuleInterfaceRequirement(
name="addresses.lookup",
version_min="0.1.0",
version_max_exclusive="0.2.0",
optional=True,
),
),
permissions=PERMISSIONS,
route_factory=_mail_router,
@@ -120,6 +126,8 @@ manifest = ModuleManifest(
retirement_provider=drop_table_retirement_provider(
mail_models.MailServerProfile,
mail_models.MailProfilePolicy,
mail_models.MailMailboxFolderIndex,
mail_models.MailMailboxMessageIndex,
label="Mail",
),
retirement_notes="Destructive retirement drops mail-owned database tables after the installer captures a database snapshot.",
@@ -128,6 +136,8 @@ manifest = ModuleManifest(
persistent_table_uninstall_guard(
mail_models.MailServerProfile,
mail_models.MailProfilePolicy,
mail_models.MailMailboxFolderIndex,
mail_models.MailMailboxMessageIndex,
label="Mail",
),
),

112
src/govoplan_mail/backend/migrations/dev_versions/4e5f708192ab_mail_mailbox_index.py Normal file
View File

@@ -0,0 +1,112 @@
"""mail mailbox index
Revision ID: 4e5f708192ab
Revises: 3d4e5f708192
Create Date: 2026-07-14 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "4e5f708192ab"
down_revision = "3d4e5f708192"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_folder_index" not in tables:
op.create_table(
"mail_mailbox_folder_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("message_count", sa.Integer(), nullable=True),
sa.Column("unseen_count", sa.Integer(), nullable=True),
sa.Column("uidvalidity", sa.String(length=255), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_folder_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_folder_index")),
sa.UniqueConstraint("profile_id", "folder", name="uq_mail_mailbox_folder_index_profile_folder"),
)
op.create_index(op.f("ix_mail_mailbox_folder_index_folder"), "mail_mailbox_folder_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), "mail_mailbox_folder_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_profile_id"), "mail_mailbox_folder_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), "mail_mailbox_folder_index", ["tenant_id"], unique=False)
op.create_index("ix_mail_mailbox_folder_index_tenant_profile", "mail_mailbox_folder_index", ["tenant_id", "profile_id"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_folder_index")}
if "message_indexed_at" not in columns:
op.add_column("mail_mailbox_folder_index", sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True))
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
if "mail_mailbox_message_index" not in tables:
op.create_table(
"mail_mailbox_message_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("uid", sa.String(length=255), nullable=False),
sa.Column("uid_int", sa.BigInteger(), nullable=False),
sa.Column("sort_position", sa.BigInteger(), nullable=False),
sa.Column("subject", sa.Text(), nullable=True),
sa.Column("from_header", sa.Text(), nullable=True),
sa.Column("to_header", sa.Text(), nullable=True),
sa.Column("cc_header", sa.Text(), nullable=True),
sa.Column("date", sa.String(length=255), nullable=True),
sa.Column("message_id", sa.Text(), nullable=True),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("body_preview", sa.Text(), nullable=True),
sa.Column("attachment_count", sa.Integer(), nullable=False),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_message_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_message_index")),
sa.UniqueConstraint("profile_id", "folder", "uid", name="uq_mail_mailbox_message_index_profile_folder_uid"),
)
op.create_index(op.f("ix_mail_mailbox_message_index_folder"), "mail_mailbox_message_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_indexed_at"), "mail_mailbox_message_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_profile_id"), "mail_mailbox_message_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_tenant_id"), "mail_mailbox_message_index", ["tenant_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_uid_int"), "mail_mailbox_message_index", ["uid_int"], unique=False)
op.create_index("ix_mail_mailbox_message_index_page", "mail_mailbox_message_index", ["tenant_id", "profile_id", "folder", "sort_position"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_message_index")}
if "sort_position" not in columns:
op.add_column("mail_mailbox_message_index", sa.Column("sort_position", sa.BigInteger(), nullable=False, server_default="0"))
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_message_index" in tables:
op.drop_index("ix_mail_mailbox_message_index_page", table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_uid_int"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_tenant_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_profile_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_indexed_at"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_folder"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_sort_position"), table_name="mail_mailbox_message_index")
op.drop_table("mail_mailbox_message_index")
if "mail_mailbox_folder_index" in tables:
op.drop_index("ix_mail_mailbox_folder_index_tenant_profile", table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_profile_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_folder"), table_name="mail_mailbox_folder_index")
op.drop_table("mail_mailbox_folder_index")

112
src/govoplan_mail/backend/migrations/versions/4e5f708192ab_v018_mail_mailbox_index.py Normal file
View File

@@ -0,0 +1,112 @@
"""v0.1.8 mail mailbox index
Revision ID: 4e5f708192ab
Revises: 3d4e5f708192
Create Date: 2026-07-14 00:00:00.000000
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
revision = "4e5f708192ab"
down_revision = "3d4e5f708192"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_folder_index" not in tables:
op.create_table(
"mail_mailbox_folder_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("message_count", sa.Integer(), nullable=True),
sa.Column("unseen_count", sa.Integer(), nullable=True),
sa.Column("uidvalidity", sa.String(length=255), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_folder_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_folder_index")),
sa.UniqueConstraint("profile_id", "folder", name="uq_mail_mailbox_folder_index_profile_folder"),
)
op.create_index(op.f("ix_mail_mailbox_folder_index_folder"), "mail_mailbox_folder_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), "mail_mailbox_folder_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_profile_id"), "mail_mailbox_folder_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), "mail_mailbox_folder_index", ["tenant_id"], unique=False)
op.create_index("ix_mail_mailbox_folder_index_tenant_profile", "mail_mailbox_folder_index", ["tenant_id", "profile_id"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_folder_index")}
if "message_indexed_at" not in columns:
op.add_column("mail_mailbox_folder_index", sa.Column("message_indexed_at", sa.DateTime(timezone=True), nullable=True))
op.create_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), "mail_mailbox_folder_index", ["message_indexed_at"], unique=False)
if "mail_mailbox_message_index" not in tables:
op.create_table(
"mail_mailbox_message_index",
sa.Column("id", sa.String(length=36), nullable=False),
sa.Column("tenant_id", sa.String(length=36), nullable=False),
sa.Column("profile_id", sa.String(length=36), nullable=False),
sa.Column("folder", sa.String(length=255), nullable=False),
sa.Column("uid", sa.String(length=255), nullable=False),
sa.Column("uid_int", sa.BigInteger(), nullable=False),
sa.Column("sort_position", sa.BigInteger(), nullable=False),
sa.Column("subject", sa.Text(), nullable=True),
sa.Column("from_header", sa.Text(), nullable=True),
sa.Column("to_header", sa.Text(), nullable=True),
sa.Column("cc_header", sa.Text(), nullable=True),
sa.Column("date", sa.String(length=255), nullable=True),
sa.Column("message_id", sa.Text(), nullable=True),
sa.Column("flags", sa.JSON(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("body_preview", sa.Text(), nullable=True),
sa.Column("attachment_count", sa.Integer(), nullable=False),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(["profile_id"], ["mail_server_profiles.id"], name=op.f("fk_mail_mailbox_message_index_profile_id_mail_server_profiles"), ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id", name=op.f("pk_mail_mailbox_message_index")),
sa.UniqueConstraint("profile_id", "folder", "uid", name="uq_mail_mailbox_message_index_profile_folder_uid"),
)
op.create_index(op.f("ix_mail_mailbox_message_index_folder"), "mail_mailbox_message_index", ["folder"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_indexed_at"), "mail_mailbox_message_index", ["indexed_at"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_profile_id"), "mail_mailbox_message_index", ["profile_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_tenant_id"), "mail_mailbox_message_index", ["tenant_id"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
op.create_index(op.f("ix_mail_mailbox_message_index_uid_int"), "mail_mailbox_message_index", ["uid_int"], unique=False)
op.create_index("ix_mail_mailbox_message_index_page", "mail_mailbox_message_index", ["tenant_id", "profile_id", "folder", "sort_position"], unique=False)
else:
columns = {column["name"] for column in inspector.get_columns("mail_mailbox_message_index")}
if "sort_position" not in columns:
op.add_column("mail_mailbox_message_index", sa.Column("sort_position", sa.BigInteger(), nullable=False, server_default="0"))
op.create_index(op.f("ix_mail_mailbox_message_index_sort_position"), "mail_mailbox_message_index", ["sort_position"], unique=False)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables = set(inspector.get_table_names())
if "mail_mailbox_message_index" in tables:
op.drop_index("ix_mail_mailbox_message_index_page", table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_uid_int"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_tenant_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_profile_id"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_indexed_at"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_folder"), table_name="mail_mailbox_message_index")
op.drop_index(op.f("ix_mail_mailbox_message_index_sort_position"), table_name="mail_mailbox_message_index")
op.drop_table("mail_mailbox_message_index")
if "mail_mailbox_folder_index" in tables:
op.drop_index("ix_mail_mailbox_folder_index_tenant_profile", table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_tenant_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_profile_id"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_message_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_indexed_at"), table_name="mail_mailbox_folder_index")
op.drop_index(op.f("ix_mail_mailbox_folder_index_folder"), table_name="mail_mailbox_folder_index")
op.drop_table("mail_mailbox_folder_index")

419
src/govoplan_mail/backend/router.py
View File

@@ -1,13 +1,20 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, Query, status
import dataclasses
from types import SimpleNamespace
from typing import Any
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from govoplan_mail.backend.schemas import (
MailAddressLookupCandidate,
MailAddressLookupResponse,
MailConnectionTestResponse,
MailImapFolderListResponse,
MailImapFolderResponse,
MailMailboxBootstrapResponse,
MailImapTestRequest,
MailMailboxAttachmentResponse,
MailMailboxMessageDetailResponse,
@@ -33,7 +40,15 @@ from govoplan_core.core.change_sequence import (
sequence_watermark_is_expired,
)
from govoplan_core.core.pagination import KeysetCursorError, decode_keyset_cursor, encode_keyset_cursor, keyset_query_fingerprint
from govoplan_core.db.session import get_session
from govoplan_core.db.session import get_database, get_session
from govoplan_mail.backend.mailbox_index import (
begin_mailbox_refresh,
cache_mailbox_folders,
cache_mailbox_messages,
cached_mailbox_folders,
cached_mailbox_message_page,
finish_mailbox_refresh,
)
from govoplan_mail.backend.mail_profiles import (
MailProfileError,
create_mail_server_profile,
@@ -49,7 +64,8 @@ from govoplan_mail.backend.mail_profiles import (
update_mail_server_profile,
)
from govoplan_mail.backend.config import ImapConfig, SmtpConfig
from govoplan_mail.backend.sending.imap import ImapAppendError, ImapConfigurationError, get_imap_message, list_imap_folders, list_imap_messages, test_imap_login
from govoplan_mail.backend.runtime import get_registry
from govoplan_mail.backend.sending.imap import ImapAppendError, ImapConfigurationError, get_imap_message, list_imap_folders, list_imap_messages, load_imap_mailbox_bootstrap, test_imap_login
from govoplan_mail.backend.sending.smtp import test_smtp_login
router = APIRouter(prefix="/mail", tags=["mail"])
@@ -62,6 +78,39 @@ MAIL_PROFILE_RESOURCE = "mail_profile"
MAIL_POLICY_RESOURCE = "mail_profile_policy"
MAILBOX_MESSAGES_CURSOR_SCOPE = "mail.mailbox.messages.v1"
DEFAULT_MAILBOX_MESSAGE_LIMIT = 50
CAPABILITY_ADDRESSES_LOOKUP = "addresses.lookup"
def _capability_payload(value: object) -> dict[str, Any]:
if dataclasses.is_dataclass(value):
return dataclasses.asdict(value)
if isinstance(value, dict):
return dict(value)
payload: dict[str, Any] = {}
for key in (
"contact_id",
"address_book_id",
"display_name",
"email",
"email_label",
"organization",
"role_title",
"tags",
"source_kind",
"source_ref",
"source_revision",
"provenance",
):
if hasattr(value, key):
payload[key] = getattr(value, key)
return payload
def _registry_capability(name: str) -> object | None:
registry = get_registry()
if registry is None or not hasattr(registry, "has_capability") or not registry.has_capability(name):
return None
return registry.capability(name)
def _require_scope(principal: ApiPrincipal, scope: str) -> None:
@@ -338,6 +387,132 @@ def _next_mailbox_cursor(
)
def _mailbox_folder_response(
result,
*,
from_cache: bool = False,
refreshing: bool = False,
indexed_at=None,
) -> MailImapFolderListResponse:
folders = [MailImapFolderResponse(name=item.name, flags=item.flags, message_count=item.message_count, unseen_count=item.unseen_count) for item in result.folders]
return MailImapFolderListResponse(
ok=True,
host=result.host,
port=result.port,
security=result.security,
message=f"Found {len(folders)} IMAP folder(s).",
folders=folders,
detected_sent_folder=result.detected_sent_folder,
from_cache=from_cache,
refreshing=refreshing,
indexed_at=indexed_at,
)
def _mailbox_messages_response(
*,
profile_id: str,
folder: str,
host: str | None,
port: int | None,
security: str | None,
total_count: int,
offset: int,
limit: int,
cursor: str | None,
next_cursor: str | None,
cursor_stable: bool,
full: bool,
messages,
from_cache: bool = False,
refreshing: bool = False,
indexed_at=None,
) -> MailMailboxMessageListResponse:
return MailMailboxMessageListResponse(
profile_id=profile_id,
folder=folder,
host=host,
port=port,
security=security,
total_count=total_count,
offset=offset,
limit=limit,
cursor=cursor,
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=full,
from_cache=from_cache,
refreshing=refreshing,
indexed_at=indexed_at,
messages=[_mailbox_summary_response(message) for message in messages],
)
def _schedule_mailbox_refresh(
background_tasks: BackgroundTasks,
*,
tenant_id: str,
profile_id: str,
folder: str,
limit: int,
offset: int = 0,
include_folder_status: bool = False,
) -> bool:
if not begin_mailbox_refresh(tenant_id, profile_id, folder):
return False
background_tasks.add_task(
_refresh_mailbox_index_task,
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
limit=limit,
offset=offset,
include_folder_status=include_folder_status,
)
return True
def _refresh_mailbox_index_task(
*,
tenant_id: str,
profile_id: str,
folder: str,
limit: int,
offset: int,
include_folder_status: bool,
) -> None:
try:
with get_database().session() as session:
imap = _imap_config_for_profile(session, tenant_id=tenant_id, profile_id=profile_id)
result = load_imap_mailbox_bootstrap(
imap_config=imap,
folder=folder,
limit=limit,
offset=offset,
include_folder_status=include_folder_status,
)
cache_mailbox_folders(session, tenant_id=tenant_id, profile_id=profile_id, result=result.folders)
cache_mailbox_messages(session, tenant_id=tenant_id, profile_id=profile_id, result=result.messages)
session.commit()
except Exception:
# Background refresh is opportunistic. Foreground requests will surface
# concrete IMAP errors when no usable cache is available.
pass
finally:
finish_mailbox_refresh(tenant_id, profile_id, folder)
def _cached_folder_selection(folders, requested: str, detected_sent_folder: str | None = None) -> str:
names = {folder.name for folder in folders}
if requested in names:
return requested
if "INBOX" in names:
return "INBOX"
if detected_sent_folder and detected_sent_folder in names:
return detected_sent_folder
return folders[0].name if folders else requested
def _imap_config_for_profile(session: Session, *, tenant_id: str, profile_id: str):
profile = get_mail_server_profile(session, tenant_id=tenant_id, profile_id=profile_id, require_active=True)
imap = imap_config_from_profile(profile)
@@ -378,6 +553,24 @@ def _full_mail_settings_delta_response(
)
@router.get("/address-lookup", response_model=MailAddressLookupResponse)
def lookup_mail_addresses(
query: str = Query(min_length=1),
limit: int = Query(default=25, ge=1, le=100),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> MailAddressLookupResponse:
_require_scope(principal, "mail:profile:use")
capability = _registry_capability(CAPABILITY_ADDRESSES_LOOKUP)
if capability is None or not hasattr(capability, "lookup"):
return MailAddressLookupResponse(available=False, candidates=[])
candidates = getattr(capability, "lookup")(session, principal, query=query, limit=limit)
return MailAddressLookupResponse(
available=True,
candidates=[MailAddressLookupCandidate.model_validate(_capability_payload(candidate)) for candidate in candidates],
)
@router.get("/settings/delta", response_model=MailSettingsDeltaResponse)
def mail_settings_delta(
scope_type: str = Query(default="tenant"),
@@ -711,8 +904,7 @@ def list_profile_imap_folders(
if imap is None:
raise MailProfileError("Mail-server profile has no IMAP configuration")
result = list_imap_folders(imap_config=imap)
folders = [MailImapFolderResponse(name=item.name, flags=item.flags, message_count=item.message_count, unseen_count=item.unseen_count) for item in result.folders]
return MailImapFolderListResponse(ok=True, host=result.host, port=result.port, security=result.security, message=f"Found {len(folders)} IMAP folder(s).", folders=folders, detected_sent_folder=result.detected_sent_folder)
return _mailbox_folder_response(result)
except MailProfileError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
except Exception as exc:
@@ -722,15 +914,169 @@ def list_profile_imap_folders(
@router.get("/profiles/{profile_id}/mailbox/folders", response_model=MailImapFolderListResponse)
def list_profile_mailbox_folders(
profile_id: str,
background_tasks: BackgroundTasks,
include_status: bool = Query(default=False),
refresh: bool = False,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_mailbox_read_scope(principal)
try:
imap = _imap_config_for_profile(session, tenant_id=principal.tenant_id, profile_id=profile_id)
result = list_imap_folders(imap_config=imap)
folders = [MailImapFolderResponse(name=item.name, flags=item.flags, message_count=item.message_count, unseen_count=item.unseen_count) for item in result.folders]
return MailImapFolderListResponse(ok=True, host=result.host, port=result.port, security=result.security, message=f"Found {len(folders)} IMAP folder(s).", folders=folders, detected_sent_folder=result.detected_sent_folder)
if not include_status and not refresh:
cached = cached_mailbox_folders(session, tenant_id=principal.tenant_id, profile_id=profile_id)
if cached is not None:
refreshing = cached.stale and _schedule_mailbox_refresh(
background_tasks,
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder="INBOX",
limit=DEFAULT_MAILBOX_MESSAGE_LIMIT,
include_folder_status=include_status,
)
result = SimpleNamespace(
host=imap.host,
port=imap.port,
security=imap.security.value,
folders=cached.folders,
detected_sent_folder=None,
)
return _mailbox_folder_response(result, from_cache=True, refreshing=refreshing, indexed_at=cached.indexed_at)
result = list_imap_folders(imap_config=imap, include_status=include_status)
cache_mailbox_folders(session, tenant_id=principal.tenant_id, profile_id=profile_id, result=result)
session.commit()
return _mailbox_folder_response(result)
except MailProfileError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
except ImapConfigurationError as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except ImapAppendError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
@router.get("/profiles/{profile_id}/mailbox/bootstrap", response_model=MailMailboxBootstrapResponse)
def bootstrap_profile_mailbox(
profile_id: str,
background_tasks: BackgroundTasks,
folder: str = Query(default="INBOX", min_length=1, max_length=255),
limit: int = Query(default=DEFAULT_MAILBOX_MESSAGE_LIMIT, ge=1, le=100),
offset: int = Query(default=0, ge=0, le=100000),
refresh: bool = False,
include_status: bool = Query(default=False),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
) -> MailMailboxBootstrapResponse:
_require_mailbox_read_scope(principal)
try:
imap = _imap_config_for_profile(session, tenant_id=principal.tenant_id, profile_id=profile_id)
if not refresh:
cached_folders = cached_mailbox_folders(session, tenant_id=principal.tenant_id, profile_id=profile_id)
if cached_folders is not None:
selected_folder = _cached_folder_selection(cached_folders.folders, folder)
cached_messages = cached_mailbox_message_page(
session,
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=selected_folder,
limit=limit,
offset=offset,
)
if cached_messages is not None:
refreshing = (cached_folders.stale or cached_messages.stale) and _schedule_mailbox_refresh(
background_tasks,
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=selected_folder,
limit=limit,
offset=offset,
include_folder_status=include_status,
)
folder_result = SimpleNamespace(
host=imap.host,
port=imap.port,
security=imap.security.value,
folders=cached_folders.folders,
detected_sent_folder=None,
)
next_cursor, cursor_stable = _next_mailbox_cursor(
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=cached_messages.folder,
limit=cached_messages.limit,
offset=cached_messages.offset,
total_count=cached_messages.total_count,
uidvalidity=cached_messages.uidvalidity,
messages=cached_messages.messages,
)
return MailMailboxBootstrapResponse(
profile_id=profile_id,
folder=cached_messages.folder,
folders=_mailbox_folder_response(
folder_result,
from_cache=True,
refreshing=refreshing,
indexed_at=cached_folders.indexed_at,
),
messages=_mailbox_messages_response(
profile_id=profile_id,
folder=cached_messages.folder,
host=imap.host,
port=imap.port,
security=imap.security.value,
total_count=cached_messages.total_count,
offset=cached_messages.offset,
limit=cached_messages.limit,
cursor=None,
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=not cursor_stable,
messages=cached_messages.messages,
from_cache=True,
refreshing=refreshing,
indexed_at=cached_messages.indexed_at,
),
)
result = load_imap_mailbox_bootstrap(
imap_config=imap,
folder=folder,
limit=limit,
offset=offset,
include_folder_status=include_status,
)
cache_mailbox_folders(session, tenant_id=principal.tenant_id, profile_id=profile_id, result=result.folders)
cache_mailbox_messages(session, tenant_id=principal.tenant_id, profile_id=profile_id, result=result.messages)
session.commit()
next_cursor, cursor_stable = _next_mailbox_cursor(
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=result.messages.folder,
limit=result.messages.limit,
offset=result.messages.offset,
total_count=result.messages.total_count,
uidvalidity=result.messages.uidvalidity,
messages=result.messages.messages,
)
return MailMailboxBootstrapResponse(
profile_id=profile_id,
folder=result.messages.folder,
folders=_mailbox_folder_response(result.folders),
messages=_mailbox_messages_response(
profile_id=profile_id,
folder=result.messages.folder,
host=result.messages.host,
port=result.messages.port,
security=result.messages.security,
total_count=result.messages.total_count,
offset=result.messages.offset,
limit=result.messages.limit,
cursor=None,
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=not cursor_stable,
messages=result.messages.messages,
),
)
except MailProfileError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
except ImapConfigurationError as exc:
@@ -742,10 +1088,12 @@ def list_profile_mailbox_folders(
@router.get("/profiles/{profile_id}/mailbox/messages", response_model=MailMailboxMessageListResponse)
def list_profile_mailbox_messages(
profile_id: str,
background_tasks: BackgroundTasks,
folder: str = Query(default="INBOX", min_length=1, max_length=255),
limit: int | None = Query(default=None, ge=1, le=100),
offset: int = Query(default=0, ge=0, le=100000),
cursor: str | None = None,
refresh: bool = False,
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
@@ -755,6 +1103,55 @@ def list_profile_mailbox_messages(
effective_limit = _mailbox_cursor_limit(cursor, limit)
fingerprint = _mailbox_cursor_fingerprint(tenant_id=principal.tenant_id, profile_id=profile_id, folder=folder, limit=effective_limit)
effective_offset, after_uid, expected_uidvalidity, cursor_values = _mailbox_cursor_position(cursor, fingerprint=fingerprint, offset=offset)
if not refresh:
cached = cached_mailbox_message_page(
session,
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=folder,
limit=effective_limit,
offset=effective_offset,
)
cache_matches_cursor = cached is not None and (
cursor is None or (expected_uidvalidity is not None and cached.uidvalidity == expected_uidvalidity)
)
if cached is not None and cache_matches_cursor:
refreshing = cached.stale and _schedule_mailbox_refresh(
background_tasks,
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=folder,
limit=effective_limit,
offset=effective_offset,
)
next_cursor, cursor_stable = _next_mailbox_cursor(
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=cached.folder,
limit=cached.limit,
offset=cached.offset,
total_count=cached.total_count,
uidvalidity=cached.uidvalidity,
messages=cached.messages,
)
return _mailbox_messages_response(
profile_id=profile_id,
folder=cached.folder,
host=imap.host,
port=imap.port,
security=imap.security.value,
total_count=cached.total_count,
offset=cached.offset,
limit=cached.limit,
cursor=cursor,
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=not cursor_stable,
messages=cached.messages,
from_cache=True,
refreshing=refreshing,
indexed_at=cached.indexed_at,
)
result = list_imap_messages(
imap_config=imap,
folder=folder,
@@ -763,6 +1160,8 @@ def list_profile_mailbox_messages(
after_uid=after_uid,
expected_uidvalidity=expected_uidvalidity,
)
cache_mailbox_messages(session, tenant_id=principal.tenant_id, profile_id=profile_id, result=result)
session.commit()
full = bool(cursor_values is not None and result.cursor_reset)
next_cursor, cursor_stable = _next_mailbox_cursor(
tenant_id=principal.tenant_id,
@@ -774,7 +1173,7 @@ def list_profile_mailbox_messages(
uidvalidity=result.uidvalidity,
messages=result.messages,
)
return MailMailboxMessageListResponse(
return _mailbox_messages_response(
profile_id=profile_id,
folder=result.folder,
host=result.host,
@@ -787,7 +1186,7 @@ def list_profile_mailbox_messages(
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=full or not cursor_stable,
messages=[_mailbox_summary_response(message) for message in result.messages],
messages=result.messages,
)
except MailProfileError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc

31
src/govoplan_mail/backend/runtime.py
View File

@@ -1,29 +1,10 @@
from __future__ import annotations
from typing import Any
from govoplan_core.core.runtime import ModuleRuntimeState
_runtime_settings: object | None = None
_runtime = ModuleRuntimeState("Mail")
def configure_runtime(*, settings: object | None = None) -> None:
global _runtime_settings
if settings is not None:
_runtime_settings = settings
def get_settings() -> object:
if _runtime_settings is not None:
return _runtime_settings
try:
from govoplan_core.settings import settings as legacy_settings
except ModuleNotFoundError as exc:
raise RuntimeError("GovOPlaN Mail runtime settings are not configured") from exc
return legacy_settings
class SettingsProxy:
def __getattr__(self, name: str) -> Any:
return getattr(get_settings(), name)
settings = SettingsProxy()
configure_runtime = _runtime.configure_runtime
get_registry = _runtime.get_registry
get_settings = _runtime.get_settings
settings = _runtime.settings

65
src/govoplan_mail/backend/schemas.py
View File

@@ -6,7 +6,14 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_mail.backend.config import ImapConfig, ImapServerConfig, SmtpConfig, SmtpServerConfig, TransportCredentials
from govoplan_mail.backend.config import (
ImapConfig,
ImapServerConfig,
SmtpConfig,
SmtpServerConfig,
TransportCredentials,
normalize_split_transport_credentials,
)
class MailSmtpTestRequest(SmtpConfig):
@@ -29,28 +36,7 @@ class MailServerProfileCredentialsPayload(BaseModel):
def _normalize_profile_transport_payload(value: object) -> object:
if not isinstance(value, dict):
return value
data = dict(value)
credentials = data.get("credentials") if isinstance(data.get("credentials"), dict) else {}
credentials = {key: dict(item) for key, item in credentials.items() if isinstance(item, dict)}
for protocol in ("smtp", "imap"):
transport = data.get(protocol)
if not isinstance(transport, dict):
continue
next_transport = dict(transport)
next_credentials = dict(credentials.get(protocol) or {})
for field in ("username", "password"):
if field in next_transport and field not in next_credentials:
next_credentials[field] = next_transport[field]
next_transport.pop(field, None)
next_transport.pop("enabled", None)
data[protocol] = next_transport
if next_credentials:
credentials[protocol] = next_credentials
if credentials:
data["credentials"] = credentials
return data
return normalize_split_transport_credentials(value)
def _merge_transport_credentials(server: SmtpServerConfig | ImapServerConfig, credentials: MailTransportCredentialsPayload) -> dict[str, object]:
@@ -213,6 +199,26 @@ class MailSettingsDeltaResponse(BaseModel):
full: bool = False
class MailAddressLookupCandidate(BaseModel):
contact_id: str
address_book_id: str
display_name: str
email: str | None = None
email_label: str | None = None
organization: str | None = None
role_title: str | None = None
tags: list[str] = Field(default_factory=list)
source_kind: str = "local"
source_ref: str | None = None
source_revision: str | None = None
provenance: dict[str, Any] = Field(default_factory=dict)
class MailAddressLookupResponse(BaseModel):
available: bool = False
candidates: list[MailAddressLookupCandidate] = Field(default_factory=list)
class MailConnectionTestResponse(BaseModel):
ok: bool
protocol: Literal["smtp", "imap"]
@@ -239,6 +245,9 @@ class MailImapFolderListResponse(BaseModel):
message: str
folders: list[MailImapFolderResponse] = Field(default_factory=list)
detected_sent_folder: str | None = None
from_cache: bool = False
refreshing: bool = False
indexed_at: datetime | None = None
details: dict[str, Any] = Field(default_factory=dict)
class MailMailboxAttachmentResponse(BaseModel):
@@ -282,9 +291,19 @@ class MailMailboxMessageListResponse(BaseModel):
next_cursor: str | None = None
cursor_stable: bool = False
full: bool = False
from_cache: bool = False
refreshing: bool = False
indexed_at: datetime | None = None
messages: list[MailMailboxMessageSummaryResponse] = Field(default_factory=list)
class MailMailboxBootstrapResponse(BaseModel):
profile_id: str
folder: str
folders: MailImapFolderListResponse
messages: MailMailboxMessageListResponse
class MailMailboxMessageResponse(BaseModel):
profile_id: str
folder: str

266
src/govoplan_mail/backend/sending/imap.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import imaplib
import logging
import re
import socket
import ssl
@@ -21,6 +22,8 @@ from govoplan_mail.backend.dev.mock_mailbox import (
record_imap_append,
)
logger = logging.getLogger(__name__)
class ImapConfigurationError(ValueError):
"""Raised when IMAP settings are incomplete or inconsistent."""
@@ -70,6 +73,10 @@ class ImapMailboxAttachmentInfo:
size_bytes: int
def _log_imap_cleanup_failure(action: str, exc: BaseException) -> None:
logger.debug("IMAP cleanup failed while %s: %s", action, exc, exc_info=True)
@dataclass(frozen=True, slots=True)
class ImapMailboxMessageSummary:
uid: str
@@ -119,6 +126,12 @@ class ImapMailboxMessageListResult:
cursor_reset: bool = False
@dataclass(frozen=True, slots=True)
class ImapMailboxBootstrapResult:
folders: ImapFolderListResult
messages: ImapMailboxMessageListResult
@dataclass(frozen=True, slots=True)
class ImapMailboxMessageResult:
host: str
@@ -170,8 +183,8 @@ def _open_imap(config: ImapConfig) -> imaplib.IMAP4:
except Exception:
try:
client.logout() # type: ignore[possibly-undefined]
except Exception:
pass
except Exception as cleanup_exc:
_log_imap_cleanup_failure("opening connection", cleanup_exc)
raise
@@ -306,58 +319,200 @@ def test_imap_login(*, imap_config: ImapConfig) -> ImapLoginTestResult:
finally:
try:
client.logout()
except Exception:
pass
except Exception as cleanup_exc:
_log_imap_cleanup_failure("testing login", cleanup_exc)
def list_imap_folders(*, imap_config: ImapConfig) -> ImapFolderListResult:
def _mock_imap_folders(*, imap_config: ImapConfig) -> ImapFolderListResult:
host, port = _require_imap_config(imap_config)
records = list_records(limit=500)
folders = []
for item in MOCK_IMAP_FOLDERS:
name = str(item["name"])
count = sum(1 for record in records if _mock_folder_matches(record, name))
folders.append(ImapMailboxInfo(name=name, flags=list(item.get("flags") or []), message_count=count, unseen_count=None))
return ImapFolderListResult(
host=host,
port=port,
security=imap_config.security.value,
folders=folders,
detected_sent_folder="Sent",
)
def _list_imap_folders_on_client(
client: imaplib.IMAP4,
*,
host: str,
port: int,
security: str,
include_status: bool,
) -> ImapFolderListResult:
typ, data = client.list()
if typ != "OK":
raise ImapAppendError(f"IMAP folder listing failed: {data!r}", temporary=True)
parsed: list[tuple[str, set[str]]] = []
folders: list[ImapMailboxInfo] = []
for item in data or []:
extracted = _extract_mailbox_name(item)
if not extracted:
continue
name, flags = extracted
parsed.append((name, flags))
message_count, unseen_count = (
(None, None)
if not include_status or _has_folder_flag(flags, "noselect")
else _imap_folder_status(client, name)
)
folders.append(ImapMailboxInfo(name=name, flags=sorted(flags), message_count=message_count, unseen_count=unseen_count))
return ImapFolderListResult(
host=host,
port=port,
security=security,
folders=folders,
detected_sent_folder=_detect_sent_folder(parsed),
)
def list_imap_folders(*, imap_config: ImapConfig, include_status: bool = True) -> ImapFolderListResult:
"""Return folders visible through IMAP LIST and the best sent-folder guess."""
host, port = _require_imap_config(imap_config)
if is_mock_imap_host(imap_config.host):
records = list_records(limit=500)
folders = []
for item in MOCK_IMAP_FOLDERS:
name = str(item["name"])
count = sum(1 for record in records if _mock_folder_matches(record, name))
folders.append(ImapMailboxInfo(name=name, flags=list(item.get("flags") or []), message_count=count, unseen_count=None))
return ImapFolderListResult(
host=host,
port=port,
security=imap_config.security.value,
folders=folders,
detected_sent_folder="Sent",
)
return _mock_imap_folders(imap_config=imap_config)
client = _open_imap(imap_config)
try:
typ, data = client.list()
if typ != "OK":
raise ImapAppendError(f"IMAP folder listing failed: {data!r}", temporary=True)
parsed: list[tuple[str, set[str]]] = []
folders: list[ImapMailboxInfo] = []
for item in data or []:
extracted = _extract_mailbox_name(item)
if not extracted:
continue
name, flags = extracted
parsed.append((name, flags))
message_count, unseen_count = (None, None) if _has_folder_flag(flags, "noselect") else _imap_folder_status(client, name)
folders.append(ImapMailboxInfo(name=name, flags=sorted(flags), message_count=message_count, unseen_count=unseen_count))
return ImapFolderListResult(
return _list_imap_folders_on_client(
client,
host=host,
port=port,
security=imap_config.security.value,
folders=folders,
detected_sent_folder=_detect_sent_folder(parsed),
include_status=include_status,
)
finally:
try:
client.logout()
except Exception:
pass
except Exception as cleanup_exc:
_log_imap_cleanup_failure("listing folders", cleanup_exc)
def _preferred_mailbox_folder(folders: list[ImapMailboxInfo], requested: str | None, detected_sent_folder: str | None) -> str:
folder_names = {folder.name for folder in folders}
requested = (requested or "").strip()
if requested and requested in folder_names:
return requested
if "INBOX" in folder_names:
return "INBOX"
if detected_sent_folder and detected_sent_folder in folder_names:
return detected_sent_folder
return folders[0].name if folders else (requested or "INBOX")
def load_imap_mailbox_bootstrap(
*,
imap_config: ImapConfig,
folder: str = "INBOX",
limit: int = 50,
offset: int = 0,
include_folder_status: bool = False,
) -> ImapMailboxBootstrapResult:
"""Load folders and one message page through a single IMAP connection."""
host, port = _require_imap_config(imap_config)
if is_mock_imap_host(imap_config.host):
folders = _mock_imap_folders(imap_config=imap_config)
selected_folder = _preferred_mailbox_folder(folders.folders, folder, folders.detected_sent_folder)
messages = list_imap_messages(imap_config=imap_config, folder=selected_folder, limit=limit, offset=offset)
return ImapMailboxBootstrapResult(folders=folders, messages=messages)
client = _open_imap(imap_config)
try:
folders = _list_imap_folders_on_client(
client,
host=host,
port=port,
security=imap_config.security.value,
include_status=include_folder_status,
)
selected_folder = _preferred_mailbox_folder(folders.folders, folder, folders.detected_sent_folder)
selected_folder, limit, offset = _normalize_mailbox_page(folder=selected_folder, limit=limit, offset=offset)
messages = _list_imap_messages_on_client(
client,
host=host,
port=port,
security=imap_config.security.value,
folder=selected_folder,
limit=limit,
offset=offset,
after_uid=None,
expected_uidvalidity=None,
)
return ImapMailboxBootstrapResult(folders=folders, messages=messages)
finally:
try:
client.logout()
except Exception as cleanup_exc:
_log_imap_cleanup_failure("bootstrapping mailbox", cleanup_exc)
def _list_imap_messages_on_client(
client: imaplib.IMAP4,
*,
host: str,
port: int,
security: str,
folder: str,
limit: int,
offset: int,
after_uid: str | None,
expected_uidvalidity: str | None,
) -> ImapMailboxMessageListResult:
total_count, uidvalidity = _select_readonly(client, folder)
if after_uid is None and expected_uidvalidity is None:
page_sequences = _paged_descending_sequences(total_count, offset=offset, limit=limit)
messages = _fetch_message_summaries_by_sequence(client, page_sequences, folder)
return ImapMailboxMessageListResult(
host=host,
port=port,
security=security,
folder=folder,
messages=messages,
total_count=total_count,
offset=offset,
limit=limit,
uidvalidity=None,
cursor_reset=False,
)
uids = _search_message_uids(client)
cursor_reset = False
effective_after_uid = after_uid
effective_offset = offset
if expected_uidvalidity and expected_uidvalidity != uidvalidity:
effective_after_uid = None
effective_offset = 0
cursor_reset = True
page_uids, effective_offset, anchor_missing = _paged_descending_uids(
uids,
offset=effective_offset,
limit=limit,
after_uid=effective_after_uid,
)
messages = _fetch_message_summaries_by_uid(client, page_uids, folder)
return ImapMailboxMessageListResult(
host=host,
port=port,
security=security,
folder=folder,
messages=messages,
total_count=len(uids) if uids else total_count,
offset=effective_offset,
limit=limit,
uidvalidity=uidvalidity,
cursor_reset=cursor_reset or anchor_missing,
)
def _has_folder_flag(flags: set[str], flag: str) -> bool:
@@ -691,9 +846,7 @@ def list_imap_messages(
"""List mailbox messages without mutating read/unread state."""
host, port = _require_imap_config(imap_config)
folder = (folder or "INBOX").strip() or "INBOX"
limit = max(1, min(limit, 100))
offset = max(0, offset)
folder, limit, offset = _normalize_mailbox_page(folder=folder, limit=limit, offset=offset)
if is_mock_imap_host(imap_config.host):
records = [record for record in list_records(limit=500) if _mock_folder_matches(record, folder)]
cursor_reset = False
@@ -736,6 +889,21 @@ def list_imap_messages(
client = _open_imap(imap_config)
try:
total_count, uidvalidity = _select_readonly(client, folder)
if after_uid is None and expected_uidvalidity is None:
page_sequences = _paged_descending_sequences(total_count, offset=offset, limit=limit)
messages = _fetch_message_summaries_by_sequence(client, page_sequences, folder)
return ImapMailboxMessageListResult(
host=host,
port=port,
security=imap_config.security.value,
folder=folder,
messages=messages,
total_count=total_count,
offset=offset,
limit=limit,
uidvalidity=None,
cursor_reset=False,
)
uids = _search_message_uids(client)
cursor_reset = False
effective_after_uid = after_uid
@@ -766,8 +934,12 @@ def list_imap_messages(
finally:
try:
client.logout()
except Exception:
pass
except Exception as cleanup_exc:
_log_imap_cleanup_failure("listing messages", cleanup_exc)
def _normalize_mailbox_page(*, folder: str | None, limit: int, offset: int) -> tuple[str, int, int]:
return (folder or "INBOX").strip() or "INBOX", max(1, min(limit, 100)), max(0, offset)
def _paged_descending_sequences(total_count: int, *, offset: int, limit: int) -> list[str]:
@@ -809,8 +981,8 @@ def get_imap_message(*, imap_config: ImapConfig, folder: str, uid: str) -> ImapM
finally:
try:
client.logout()
except Exception:
pass
except Exception as cleanup_exc:
_log_imap_cleanup_failure("reading message", cleanup_exc)
def append_message_to_sent(
@@ -867,5 +1039,5 @@ def append_message_to_sent(
if client is not None:
try:
client.logout()
except Exception:
pass
except Exception as cleanup_exc:
_log_imap_cleanup_failure("appending sent message", cleanup_exc)

4
src/govoplan_mail/backend/sending/rate_limit.py
View File

@@ -40,8 +40,8 @@ def wait_for_rate_limit(*, key: str, messages_per_minute: int, enabled: bool = T
if not enabled or not _distributed_rate_limit_enabled():
return RateLimitDecision(key=key, messages_per_minute=messages_per_minute, gap_seconds=gap, waited_seconds=0.0)
redis_key = f"multimailer:ratelimit:{key}:next_allowed"
lock_key = f"multimailer:ratelimit:{key}:lock"
redis_key = f"govoplan:ratelimit:{key}:next_allowed"
lock_key = f"govoplan:ratelimit:{key}:lock"
waited = 0.0
try:

156
src/govoplan_mail/backend/sending/smtp.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import copy
import logging
import smtplib
import ssl
from dataclasses import dataclass
@@ -15,6 +16,8 @@ from govoplan_mail.backend.dev.mock_mailbox import (
record_smtp_delivery,
)
logger = logging.getLogger(__name__)
class SmtpConfigurationError(ValueError):
"""Raised when SMTP settings are incomplete or inconsistent."""
@@ -56,6 +59,10 @@ class SmtpSendResult:
return len(self.envelope_recipients) - len(self.refused_recipients)
def _log_smtp_cleanup_failure(action: str, exc: BaseException) -> None:
logger.debug("SMTP cleanup failed while %s: %s", action, exc, exc_info=True)
def _require_smtp_config(config: SmtpConfig) -> tuple[str, int]:
if not config.host:
raise SmtpConfigurationError("SMTP host is required")
@@ -89,8 +96,8 @@ def _open_smtp(config: SmtpConfig) -> smtplib.SMTP:
# on GC, but explicit cleanup is safer when the variable exists.
try:
smtp.quit() # type: ignore[possibly-undefined]
except Exception:
pass
except Exception as cleanup_exc:
_log_smtp_cleanup_failure("opening connection", cleanup_exc)
raise
@@ -134,11 +141,12 @@ def test_smtp_login(*, smtp_config: SmtpConfig) -> SmtpLoginTestResult:
finally:
try:
smtp.quit()
except Exception:
except Exception as quit_exc:
_log_smtp_cleanup_failure("testing login quit", quit_exc)
try:
smtp.close()
except Exception:
pass
except Exception as close_exc:
_log_smtp_cleanup_failure("testing login close", close_exc)
def prepare_test_message(
@@ -161,12 +169,12 @@ def prepare_test_message(
del test_message[header]
# Replace potential previous marker headers if the user test-sends an EML twice.
for header in ["X-MultiMailer-Test-Send"]:
for header in ["X-GovOPlaN-Test-Send"]:
if header in test_message:
del test_message[header]
test_message["To"] = formataddr((test_recipient_name or test_recipient, test_recipient))
test_message["X-MultiMailer-Test-Send"] = "true"
test_message["X-GovOPlaN-Test-Send"] = "true"
return test_message
@@ -177,43 +185,97 @@ def _send_smtp_payload(
envelope_from: str,
envelope_recipients: list[str],
) -> SmtpSendResult:
host, port, recipients = _prepare_smtp_send(
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
)
if is_mock_smtp_host(smtp_config.host):
_accepted, refused = _send_mock_smtp_payload(
message,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=recipients,
)
return _smtp_send_result(
smtp_config=smtp_config,
host=host,
port=port,
envelope_from=envelope_from,
envelope_recipients=recipients,
refused=refused,
)
refused = _send_network_smtp_payload(
message,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=recipients,
)
return _smtp_send_result(
smtp_config=smtp_config,
host=host,
port=port,
envelope_from=envelope_from,
envelope_recipients=recipients,
refused=refused,
)
def _prepare_smtp_send(
*,
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
) -> tuple[str, int, list[str]]:
host, port = _require_smtp_config(smtp_config)
if not envelope_from:
raise SmtpConfigurationError("SMTP envelope sender is required")
if not envelope_recipients:
recipients = [recipient for recipient in envelope_recipients if recipient]
if not recipients:
raise SmtpConfigurationError("at least one SMTP envelope recipient is required")
return host, port, recipients
if is_mock_smtp_host(smtp_config.host):
if consume_fail_next_smtp():
raise SmtpSendError("Mock SMTP configured to fail the next send")
failures = get_failures()
reject_text = str(failures.get("smtp_reject_recipients_containing") or "").strip().lower()
refused: dict[str, tuple[int, bytes]] = {}
accepted = list(envelope_recipients)
if reject_text:
refused = {
recipient: (550, b"mock recipient rejected")
for recipient in envelope_recipients
if reject_text in recipient.lower()
}
accepted = [recipient for recipient in envelope_recipients if recipient not in refused]
if not accepted:
raise SmtpSendError(f"all mock SMTP recipients were refused: {_decode_refused(refused)}")
record_smtp_delivery(
message,
envelope_from=envelope_from,
envelope_recipients=accepted,
smtp_host=smtp_config.host,
)
return SmtpSendResult(
host=host,
port=port,
security=smtp_config.security.value,
envelope_from=envelope_from,
envelope_recipients=list(envelope_recipients),
refused_recipients=_decode_refused(refused),
)
def _send_mock_smtp_payload(
message: EmailMessage | bytes,
*,
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
) -> tuple[list[str], dict[str, tuple[int, bytes]]]:
if consume_fail_next_smtp():
raise SmtpSendError("Mock SMTP configured to fail the next send")
failures = get_failures()
reject_text = str(failures.get("smtp_reject_recipients_containing") or "").strip().lower()
refused: dict[str, tuple[int, bytes]] = {}
accepted = list(envelope_recipients)
if reject_text:
refused = {
recipient: (550, b"mock recipient rejected")
for recipient in envelope_recipients
if reject_text in recipient.lower()
}
accepted = [recipient for recipient in envelope_recipients if recipient not in refused]
if not accepted:
raise SmtpSendError(f"all mock SMTP recipients were refused: {_decode_refused(refused)}")
record_smtp_delivery(
message,
envelope_from=envelope_from,
envelope_recipients=accepted,
smtp_host=smtp_config.host,
)
return accepted, refused
def _send_network_smtp_payload(
message: EmailMessage | bytes,
*,
smtp_config: SmtpConfig,
envelope_from: str,
envelope_recipients: list[str],
) -> dict[str, tuple[int, bytes]]:
try:
smtp = _open_smtp(smtp_config)
except smtplib.SMTPAuthenticationError as exc:
@@ -266,12 +328,24 @@ def _send_smtp_payload(
finally:
try:
smtp.quit()
except Exception:
except Exception as quit_exc:
_log_smtp_cleanup_failure("sending message quit", quit_exc)
try:
smtp.close()
except Exception:
pass
except Exception as close_exc:
_log_smtp_cleanup_failure("sending message close", close_exc)
return refused
def _smtp_send_result(
*,
smtp_config: SmtpConfig,
host: str,
port: int,
envelope_from: str,
envelope_recipients: list[str],
refused: dict[str, tuple[int, bytes]],
) -> SmtpSendResult:
return SmtpSendResult(
host=host,
port=port,