campaign sending prototype
This commit is contained in:
@@ -6,12 +6,18 @@ import socket
|
||||
import ssl
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from email import policy
|
||||
from email.message import EmailMessage
|
||||
from email.parser import BytesParser
|
||||
from typing import Any
|
||||
|
||||
from govoplan_mail.backend.config import ImapConfig, TransportSecurity
|
||||
from govoplan_mail.backend.dev.mock_mailbox import (
|
||||
MOCK_IMAP_FOLDERS,
|
||||
consume_fail_next_imap,
|
||||
get_record,
|
||||
is_mock_imap_host,
|
||||
list_records,
|
||||
record_imap_append,
|
||||
)
|
||||
|
||||
@@ -55,6 +61,66 @@ class ImapFolderListResult:
|
||||
detected_sent_folder: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImapMailboxAttachmentInfo:
|
||||
filename: str | None
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImapMailboxMessageSummary:
|
||||
uid: str
|
||||
folder: str
|
||||
subject: str | None
|
||||
from_header: str | None
|
||||
to_header: str | None
|
||||
cc_header: str | None
|
||||
date: str | None
|
||||
message_id: str | None
|
||||
flags: list[str]
|
||||
size_bytes: int | None
|
||||
body_preview: str | None
|
||||
attachment_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImapMailboxMessageDetail:
|
||||
uid: str
|
||||
folder: str
|
||||
subject: str | None
|
||||
from_header: str | None
|
||||
to_header: str | None
|
||||
cc_header: str | None
|
||||
date: str | None
|
||||
message_id: str | None
|
||||
flags: list[str]
|
||||
size_bytes: int | None
|
||||
body_preview: str | None
|
||||
body_text: str | None
|
||||
body_html: str | None
|
||||
headers: dict[str, str]
|
||||
attachments: list[ImapMailboxAttachmentInfo]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImapMailboxMessageListResult:
|
||||
host: str
|
||||
port: int
|
||||
security: str
|
||||
folder: str
|
||||
messages: list[ImapMailboxMessageSummary]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImapMailboxMessageResult:
|
||||
host: str
|
||||
port: int
|
||||
security: str
|
||||
folder: str
|
||||
message: ImapMailboxMessageDetail
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImapAppendResult:
|
||||
host: str
|
||||
@@ -283,6 +349,226 @@ def list_imap_folders(*, imap_config: ImapConfig) -> ImapFolderListResult:
|
||||
pass
|
||||
|
||||
|
||||
def _parse_message(raw: bytes) -> EmailMessage:
|
||||
return BytesParser(policy=policy.default).parsebytes(raw)
|
||||
|
||||
|
||||
def _header_text(message: EmailMessage, name: str) -> str | None:
|
||||
value = message.get(name)
|
||||
return str(value) if value is not None else None
|
||||
|
||||
|
||||
def _body_content(message: EmailMessage, prefer: tuple[str, ...]) -> str | None:
|
||||
try:
|
||||
body = message.get_body(preferencelist=prefer) if message.is_multipart() else message
|
||||
if body is None:
|
||||
return None
|
||||
content = body.get_content()
|
||||
except Exception:
|
||||
return None
|
||||
text = str(content).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _body_preview(message: EmailMessage) -> str | None:
|
||||
text = _body_content(message, ("plain", "html"))
|
||||
if not text:
|
||||
return None
|
||||
preview = re.sub(r"\s+", " ", text).strip()
|
||||
return preview[:600] or None
|
||||
|
||||
|
||||
def _attachment_infos(message: EmailMessage) -> list[ImapMailboxAttachmentInfo]:
|
||||
attachments: list[ImapMailboxAttachmentInfo] = []
|
||||
for part in message.iter_attachments():
|
||||
payload = part.get_payload(decode=True) or b""
|
||||
attachments.append(
|
||||
ImapMailboxAttachmentInfo(
|
||||
filename=part.get_filename(),
|
||||
content_type=part.get_content_type(),
|
||||
size_bytes=len(payload),
|
||||
)
|
||||
)
|
||||
return attachments
|
||||
|
||||
|
||||
def _message_summary_from_raw(*, uid: str, folder: str, raw: bytes, flags: list[str], size_bytes: int | None) -> ImapMailboxMessageSummary:
|
||||
message = _parse_message(raw)
|
||||
attachments = _attachment_infos(message)
|
||||
return ImapMailboxMessageSummary(
|
||||
uid=uid,
|
||||
folder=folder,
|
||||
subject=_header_text(message, "Subject"),
|
||||
from_header=_header_text(message, "From"),
|
||||
to_header=_header_text(message, "To"),
|
||||
cc_header=_header_text(message, "Cc"),
|
||||
date=_header_text(message, "Date"),
|
||||
message_id=_header_text(message, "Message-ID"),
|
||||
flags=flags,
|
||||
size_bytes=size_bytes if size_bytes is not None else len(raw),
|
||||
body_preview=_body_preview(message),
|
||||
attachment_count=len(attachments),
|
||||
)
|
||||
|
||||
|
||||
def _message_detail_from_raw(*, uid: str, folder: str, raw: bytes, flags: list[str], size_bytes: int | None) -> ImapMailboxMessageDetail:
|
||||
message = _parse_message(raw)
|
||||
attachments = _attachment_infos(message)
|
||||
return ImapMailboxMessageDetail(
|
||||
uid=uid,
|
||||
folder=folder,
|
||||
subject=_header_text(message, "Subject"),
|
||||
from_header=_header_text(message, "From"),
|
||||
to_header=_header_text(message, "To"),
|
||||
cc_header=_header_text(message, "Cc"),
|
||||
date=_header_text(message, "Date"),
|
||||
message_id=_header_text(message, "Message-ID"),
|
||||
flags=flags,
|
||||
size_bytes=size_bytes if size_bytes is not None else len(raw),
|
||||
body_preview=_body_preview(message),
|
||||
body_text=_body_content(message, ("plain",)),
|
||||
body_html=_body_content(message, ("html",)),
|
||||
headers={str(key): str(value) for key, value in message.items()},
|
||||
attachments=attachments,
|
||||
)
|
||||
|
||||
|
||||
def _parse_fetch_response(data: list[Any] | tuple[Any, ...] | None) -> tuple[str | None, list[str], int | None, bytes]:
|
||||
metadata_parts: list[str] = []
|
||||
raw: bytes | None = None
|
||||
for item in data or []:
|
||||
if isinstance(item, tuple):
|
||||
metadata_parts.append(_decode_item(item[0]))
|
||||
if isinstance(item[1], bytes):
|
||||
raw = item[1]
|
||||
else:
|
||||
metadata_parts.append(_decode_item(item))
|
||||
metadata = " ".join(part for part in metadata_parts if part)
|
||||
uid_match = re.search(r"\bUID\s+(\d+)", metadata, re.IGNORECASE)
|
||||
size_match = re.search(r"\bRFC822\.SIZE\s+(\d+)", metadata, re.IGNORECASE)
|
||||
flags_match = re.search(r"\bFLAGS\s+\(([^)]*)\)", metadata, re.IGNORECASE)
|
||||
flags = flags_match.group(1).split() if flags_match else []
|
||||
if raw is None:
|
||||
raise ImapAppendError(f"IMAP message fetch returned no message body: {metadata!r}", temporary=True)
|
||||
return (
|
||||
uid_match.group(1) if uid_match else None,
|
||||
flags,
|
||||
int(size_match.group(1)) if size_match else None,
|
||||
raw,
|
||||
)
|
||||
|
||||
|
||||
def _select_readonly(client: imaplib.IMAP4, folder: str) -> None:
|
||||
typ, data = client.select(folder, readonly=True)
|
||||
if typ != "OK":
|
||||
raise ImapAppendError(f"IMAP folder {folder!r} could not be opened read-only: {data!r}", temporary=False)
|
||||
|
||||
|
||||
def _fetch_message_by_uid(client: imaplib.IMAP4, uid: str) -> tuple[str, list[str], int | None, bytes]:
|
||||
typ, data = client.uid("fetch", str(uid), "(UID FLAGS RFC822.SIZE BODY.PEEK[])")
|
||||
if typ != "OK":
|
||||
raise ImapAppendError(f"IMAP message {uid!r} fetch failed: {data!r}", temporary=True)
|
||||
fetched_uid, flags, size_bytes, raw = _parse_fetch_response(data)
|
||||
return fetched_uid or str(uid), flags, size_bytes, raw
|
||||
|
||||
|
||||
def _mock_record_folder(record: dict[str, Any]) -> str:
|
||||
folder = str(record.get("folder") or "").strip()
|
||||
if folder:
|
||||
return folder
|
||||
return "Sent" if record.get("kind") == "imap_append" else "INBOX"
|
||||
|
||||
|
||||
def _mock_folder_matches(record: dict[str, Any], folder: str) -> bool:
|
||||
return _mock_record_folder(record).casefold() == folder.casefold()
|
||||
|
||||
|
||||
def _mock_raw_bytes(record: dict[str, Any]) -> bytes:
|
||||
raw = record.get("raw_eml")
|
||||
if isinstance(raw, str) and raw:
|
||||
return raw.encode("utf-8", errors="replace")
|
||||
headers = record.get("headers") if isinstance(record.get("headers"), dict) else {}
|
||||
lines = [f"{key}: {value}" for key, value in headers.items()]
|
||||
lines.append("")
|
||||
lines.append(str(record.get("body_preview") or ""))
|
||||
return "\n".join(lines).encode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def list_imap_messages(*, imap_config: ImapConfig, folder: str = "INBOX", limit: int = 50) -> ImapMailboxMessageListResult:
|
||||
"""List mailbox messages without mutating read/unread state."""
|
||||
|
||||
host, port = _require_imap_config(imap_config)
|
||||
folder = (folder or "INBOX").strip() or "INBOX"
|
||||
limit = max(1, min(limit, 100))
|
||||
if is_mock_imap_host(imap_config.host):
|
||||
records = [record for record in list_records(limit=500) if _mock_folder_matches(record, folder)]
|
||||
messages = [
|
||||
_message_summary_from_raw(
|
||||
uid=str(record.get("id") or ""),
|
||||
folder=_mock_record_folder(record),
|
||||
raw=_mock_raw_bytes(record),
|
||||
flags=["\\Seen"] if record.get("kind") == "imap_append" else [],
|
||||
size_bytes=int(record.get("size_bytes") or 0) or None,
|
||||
)
|
||||
for record in records[:limit]
|
||||
]
|
||||
return ImapMailboxMessageListResult(host=host, port=port, security=imap_config.security.value, folder=folder, messages=messages)
|
||||
|
||||
client = _open_imap(imap_config)
|
||||
try:
|
||||
_select_readonly(client, folder)
|
||||
typ, data = client.uid("search", None, "ALL")
|
||||
if typ != "OK":
|
||||
raise ImapAppendError(f"IMAP search failed for folder {folder!r}: {data!r}", temporary=True)
|
||||
uid_bytes = data[0] if data else b""
|
||||
uids = _decode_item(uid_bytes).split()
|
||||
messages: list[ImapMailboxMessageSummary] = []
|
||||
for uid in reversed(uids[-limit:]):
|
||||
fetched_uid, flags, size_bytes, raw = _fetch_message_by_uid(client, uid)
|
||||
messages.append(_message_summary_from_raw(uid=fetched_uid, folder=folder, raw=raw, flags=flags, size_bytes=size_bytes))
|
||||
return ImapMailboxMessageListResult(host=host, port=port, security=imap_config.security.value, folder=folder, messages=messages)
|
||||
finally:
|
||||
try:
|
||||
client.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_imap_message(*, imap_config: ImapConfig, folder: str, uid: str) -> ImapMailboxMessageResult:
|
||||
"""Fetch one message body using BODY.PEEK in a read-only selected mailbox."""
|
||||
|
||||
host, port = _require_imap_config(imap_config)
|
||||
folder = (folder or "INBOX").strip() or "INBOX"
|
||||
uid = str(uid).strip()
|
||||
if not uid:
|
||||
raise ImapConfigurationError("Message UID is required")
|
||||
|
||||
if is_mock_imap_host(imap_config.host):
|
||||
record = get_record(uid, include_raw=True)
|
||||
if not record or not _mock_folder_matches(record, folder):
|
||||
raise ImapAppendError(f"Mock mailbox message {uid!r} was not found in folder {folder!r}", temporary=False)
|
||||
message = _message_detail_from_raw(
|
||||
uid=uid,
|
||||
folder=_mock_record_folder(record),
|
||||
raw=_mock_raw_bytes(record),
|
||||
flags=["\\Seen"] if record.get("kind") == "imap_append" else [],
|
||||
size_bytes=int(record.get("size_bytes") or 0) or None,
|
||||
)
|
||||
return ImapMailboxMessageResult(host=host, port=port, security=imap_config.security.value, folder=folder, message=message)
|
||||
|
||||
client = _open_imap(imap_config)
|
||||
try:
|
||||
_select_readonly(client, folder)
|
||||
fetched_uid, flags, size_bytes, raw = _fetch_message_by_uid(client, uid)
|
||||
message = _message_detail_from_raw(uid=fetched_uid, folder=folder, raw=raw, flags=flags, size_bytes=size_bytes)
|
||||
return ImapMailboxMessageResult(host=host, port=port, security=imap_config.security.value, folder=folder, message=message)
|
||||
finally:
|
||||
try:
|
||||
client.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def append_message_to_sent(
|
||||
message_bytes: bytes,
|
||||
*,
|
||||
|
||||
@@ -21,18 +21,23 @@ def _redis_client() -> Redis:
|
||||
return Redis.from_url(settings.redis_url, decode_responses=True)
|
||||
|
||||
|
||||
def _distributed_rate_limit_enabled() -> bool:
|
||||
return bool(getattr(settings, "celery_enabled", False))
|
||||
|
||||
|
||||
def wait_for_rate_limit(*, key: str, messages_per_minute: int, enabled: bool = True) -> RateLimitDecision:
|
||||
"""Throttle sends across worker processes using Redis when available.
|
||||
|
||||
The implementation stores the next allowed send timestamp per key. A Redis
|
||||
lock keeps multiple Celery processes from reading/updating the timestamp at
|
||||
the same time. If Redis is unavailable, it falls back to no distributed wait;
|
||||
the per-container Celery concurrency still protects local development.
|
||||
the same time. Direct local development runs do not have a broker, so Redis
|
||||
is only used when Celery/worker mode is explicitly enabled. If Redis is
|
||||
unavailable, it falls back to no distributed wait.
|
||||
"""
|
||||
|
||||
messages_per_minute = max(1, int(messages_per_minute or 1))
|
||||
gap = 60.0 / messages_per_minute
|
||||
if not enabled:
|
||||
if not enabled or not _distributed_rate_limit_enabled():
|
||||
return RateLimitDecision(key=key, messages_per_minute=messages_per_minute, gap_seconds=gap, waited_seconds=0.0)
|
||||
|
||||
redis_key = f"multimailer:ratelimit:{key}:next_allowed"
|
||||
|
||||
Reference in New Issue
Block a user