mock server, file and folder management

server/app/mailer/dev/__init__.py

@@ -0,0 +1 @@
+"""Development-only mail sandbox helpers."""

server/app/mailer/dev/mock_campaign.py

@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import json
+import tempfile
+from email import policy
+from email.message import EmailMessage
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.db.models import Campaign, CampaignVersion
+from app.mailer.campaign.loader import load_campaign_config
+from app.mailer.campaign.validation import validate_campaign_config
+from app.mailer.messages.builder import build_campaign_messages
+from app.mailer.messages.models import MessageAddress, MessageDraft, MessageValidationStatus
+from app.mailer.dev.mock_mailbox import (
+    clear_records,
+    consume_fail_next_imap,
+    consume_fail_next_smtp,
+    get_failures,
+    list_records,
+    record_imap_append,
+    record_smtp_delivery,
+)
+
+
+class MockCampaignSendError(RuntimeError):
+    pass
+
+
+def _message_address_payload(address: MessageAddress | None) -> dict[str, Any] | None:
+    if address is None:
+        return None
+    return {"email": address.email, "name": address.name}
+
+
+def _message_addresses_payload(addresses: list[MessageAddress]) -> list[dict[str, Any]]:
+    return [{"email": item.email, "name": item.name} for item in addresses]
+
+
+def _issue_payloads(message: MessageDraft) -> list[dict[str, Any]]:
+    return [issue.model_dump(mode="json") for issue in message.issues]
+
+
+def _attachment_payloads(message: MessageDraft) -> list[dict[str, Any]]:
+    return [attachment.model_dump(mode="json") for attachment in message.attachments]
+
+
+def _message_payload(message: MessageDraft) -> dict[str, Any]:
+    return {
+        "entry_index": message.entry_index,
+        "entry_id": message.entry_id,
+        "active": message.active,
+        "subject": message.subject,
+        "from": _message_address_payload(message.from_),
+        "to": _message_addresses_payload(message.to),
+        "cc": _message_addresses_payload(message.cc),
+        "bcc": _message_addresses_payload(message.bcc),
+        "reply_to": _message_addresses_payload(message.reply_to),
+        "build_status": str(message.build_status.value if hasattr(message.build_status, "value") else message.build_status),
+        "validation_status": message.validation_status.value,
+        "send_status": str(message.send_status.value if hasattr(message.send_status, "value") else message.send_status),
+        "imap_status": message.imap_status.value,
+        "attachment_count": message.attachment_count,
+        "attachments": _attachment_payloads(message),
+        "issues": _issue_payloads(message),
+        "eml_size_bytes": message.eml_size_bytes,
+        "queueable": message.is_queueable,
+    }
+
+
+def _recipient_emails(message: MessageDraft) -> list[str]:
+    values = [item.email for item in message.to + message.cc + message.bcc if item.email]
+    return list(dict.fromkeys(values))
+
+
+def _envelope_from(message: MessageDraft, *, fallback: str = "mock-sender@mock.local") -> str:
+    if message.bounce_to:
+        return message.bounce_to[0].email
+    if message.from_ and message.from_.email:
+        return message.from_.email
+    return fallback
+
+
+def _raw_message_bytes(message: EmailMessage) -> bytes:
+    return message.as_bytes(policy=policy.SMTP)
+
+
+def _smtp_rejection_matches(recipients: list[str]) -> list[str]:
+    needle = (get_failures().get("smtp_reject_recipients_containing") or "").strip().lower()
+    if not needle:
+        return []
+    return [recipient for recipient in recipients if needle in recipient.lower()]
+
+
+def _can_mock_send(
+    message: MessageDraft,
+    *,
+    include_warnings: bool,
+    include_needs_review: bool,
+) -> tuple[bool, str | None]:
+    if not message.active:
+        return False, "Recipient is inactive"
+    if str(message.build_status.value if hasattr(message.build_status, "value") else message.build_status) != "built":
+        return False, f"Message is not built ({message.build_status})"
+    if message.validation_status == MessageValidationStatus.READY:
+        return True, None
+    if message.validation_status == MessageValidationStatus.WARNING and include_warnings:
+        return True, None
+    if message.validation_status == MessageValidationStatus.NEEDS_REVIEW and include_needs_review:
+        return True, None
+    return False, f"Validation status is {message.validation_status.value}"
+
+
+def run_mock_campaign_send(
+    session: Session,
+    *,
+    tenant_id: str,
+    campaign_id: str,
+    version_id: str | None = None,
+    send: bool = False,
+    include_warnings: bool = True,
+    include_needs_review: bool = False,
+    append_sent: bool = True,
+    clear_mailbox: bool = False,
+    check_files: bool = False,
+) -> dict[str, Any]:
+    """Validate, build and optionally mock-send a version without mutating it.
+
+    This is a dev/test route. It does not change campaign/version status, does
+    not queue real jobs and does not use the configured SMTP/IMAP servers. It
+    records mock SMTP deliveries and mock IMAP appends in the integrated mock
+    mailbox only when send=True.
+    """
+
+    campaign = session.query(Campaign).filter(Campaign.id == campaign_id, Campaign.tenant_id == tenant_id).one_or_none()
+    if not campaign:
+        raise MockCampaignSendError("Campaign not found or not accessible")
+    wanted_version_id = version_id or campaign.current_version_id
+    if not wanted_version_id:
+        raise MockCampaignSendError("Campaign has no current version")
+    version = session.get(CampaignVersion, wanted_version_id)
+    if not version or version.campaign_id != campaign.id:
+        raise MockCampaignSendError("Campaign version not found or not part of campaign")
+
+    if clear_mailbox:
+        clear_records()
+
+    with tempfile.TemporaryDirectory(prefix="multimailer-mock-send-") as temp_dir:
+        temp_path = Path(temp_dir)
+        campaign_path = temp_path / f"campaign-{version.id}.json"
+        campaign_path.write_text(json.dumps(version.raw_json, ensure_ascii=False, indent=2), encoding="utf-8")
+        config = load_campaign_config(campaign_path)
+        validation_report = validate_campaign_config(config, campaign_file=campaign_path, check_files=check_files)
+        build_result = build_campaign_messages(config, campaign_file=campaign_path, write_eml=False)
+
+        send_results: list[dict[str, Any]] = []
+        sent_count = 0
+        failed_count = 0
+        skipped_count = 0
+        imap_appended_count = 0
+        imap_failed_count = 0
+
+        for built in build_result.built_messages:
+            draft = built.draft
+            can_send, skip_reason = _can_mock_send(draft, include_warnings=include_warnings, include_needs_review=include_needs_review)
+            row: dict[str, Any] = {
+                "entry_index": draft.entry_index,
+                "entry_id": draft.entry_id,
+                "subject": draft.subject,
+                "validation_status": draft.validation_status.value,
+                "build_status": str(draft.build_status.value if hasattr(draft.build_status, "value") else draft.build_status),
+                "to": _message_addresses_payload(draft.to),
+                "attachments": _attachment_payloads(draft),
+                "issues": _issue_payloads(draft),
+            }
+
+            if not can_send or built.mime is None:
+                skipped_count += 1
+                row.update({"status": "skipped", "message": skip_reason or "Message has no MIME output"})
+                send_results.append(row)
+                continue
+
+            recipients = _recipient_emails(draft)
+            envelope_from = _envelope_from(draft)
+            if not recipients:
+                skipped_count += 1
+                row.update({"status": "skipped", "message": "No envelope recipients"})
+                send_results.append(row)
+                continue
+
+            if not send:
+                row.update({"status": "ready", "message": f"Would send to {len(recipients)} recipient(s)", "envelope_from": envelope_from, "envelope_recipients": recipients})
+                send_results.append(row)
+                continue
+
+            try:
+                if consume_fail_next_smtp():
+                    raise MockCampaignSendError("Configured mock failure: next SMTP delivery fails")
+                rejected = _smtp_rejection_matches(recipients)
+                if rejected and len(rejected) == len(recipients):
+                    raise MockCampaignSendError(f"Configured mock failure: all recipients rejected ({', '.join(rejected)})")
+
+                accepted = [recipient for recipient in recipients if recipient not in rejected]
+                smtp_record = record_smtp_delivery(built.mime, envelope_from=envelope_from, envelope_recipients=accepted, smtp_host="mock.smtp.local")
+                sent_count += 1
+                row.update({
+                    "status": "sent",
+                    "message": f"Mock SMTP captured as {smtp_record.id}",
+                    "smtp_message_id": smtp_record.id,
+                    "envelope_from": envelope_from,
+                    "envelope_recipients": accepted,
+                    "refused_recipients": rejected,
+                })
+
+                if append_sent:
+                    try:
+                        if consume_fail_next_imap():
+                            raise MockCampaignSendError("Configured mock failure: next IMAP append fails")
+                        folder = "Sent"
+                        if config.delivery.imap_append_sent.folder and config.delivery.imap_append_sent.folder != "auto":
+                            folder = config.delivery.imap_append_sent.folder
+                        elif config.server.imap and config.server.imap.sent_folder and config.server.imap.sent_folder != "auto":
+                            folder = config.server.imap.sent_folder
+                        imap_record = record_imap_append(_raw_message_bytes(built.mime), folder=folder, imap_host="mock.imap.local")
+                        imap_appended_count += 1
+                        row.update({"imap_status": "appended", "imap_message_id": imap_record.id, "imap_folder": folder})
+                    except Exception as exc:
+                        imap_failed_count += 1
+                        row.update({"imap_status": "failed", "imap_error": str(exc)})
+
+            except Exception as exc:
+                failed_count += 1
+                row.update({"status": "failed", "message": str(exc), "envelope_from": envelope_from, "envelope_recipients": recipients})
+
+            send_results.append(row)
+
+    validation_json = validation_report.model_dump(mode="json")
+    validation_json.update({"ok": validation_report.ok, "error_count": validation_report.error_count, "warning_count": validation_report.warning_count})
+    build_report = build_result.report
+    build_json = build_report.model_dump(mode="json")
+    build_json.update({
+        "built_count": build_report.built_count,
+        "queueable_count": build_report.queueable_count,
+        "needs_review_count": build_report.needs_review_count,
+        "blocked_count": build_report.blocked_count,
+        "warning_count": build_report.warning_count,
+        "ready_count": build_report.ready_count,
+        "messages": [_message_payload(message) for message in build_report.messages],
+    })
+
+    attempted_count = sum(1 for row in send_results if row.get("status") in {"sent", "failed"})
+    return {
+        "campaign_id": campaign.id,
+        "version_id": version.id,
+        "version_number": version.version_number,
+        "send_requested": send,
+        "include_warnings": include_warnings,
+        "include_needs_review": include_needs_review,
+        "append_sent": append_sent,
+        "steps": [
+            {"key": "validate", "label": "Validate campaign JSON", "status": "ok" if validation_report.ok else "needs_review", "summary": validation_json},
+            {"key": "build", "label": "Build messages", "status": "ok" if build_report.queueable_count else "needs_review", "summary": {"built": build_report.built_count, "queueable": build_report.queueable_count, "needs_review": build_report.needs_review_count, "blocked": build_report.blocked_count}},
+            {"key": "send", "label": "Mock SMTP delivery", "status": "skipped" if not send else ("ok" if failed_count == 0 else "needs_review"), "summary": {"attempted": attempted_count, "sent": sent_count, "failed": failed_count, "skipped": skipped_count}},
+            {"key": "imap", "label": "Mock IMAP Sent append", "status": "skipped" if not send or not append_sent else ("ok" if imap_failed_count == 0 else "needs_review"), "summary": {"appended": imap_appended_count, "failed": imap_failed_count}},
+        ],
+        "validation": validation_json,
+        "build": build_json,
+        "send": {
+            "attempted_count": attempted_count,
+            "sent_count": sent_count,
+            "failed_count": failed_count,
+            "skipped_count": skipped_count,
+            "imap_appended_count": imap_appended_count,
+            "imap_failed_count": imap_failed_count,
+            "results": send_results,
+        },
+        "mailbox": {"messages": list_records(limit=200)},
+    }

server/app/mailer/dev/mock_mailbox.py

@@ -0,0 +1,327 @@
+from __future__ import annotations
+
+import json
+import re
+import shutil
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from email import policy
+from email.message import EmailMessage
+from email.parser import BytesParser
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from app.settings import settings
+
+MOCK_SMTP_HOSTS = {"mock", "mock.smtp", "mock.smtp.local", "mock-mail", "__mock_smtp__"}
+MOCK_IMAP_HOSTS = {"mock", "mock.imap", "mock.imap.local", "mock-mail", "__mock_imap__"}
+MOCK_IMAP_FOLDERS = [
+    {"name": "INBOX", "flags": []},
+    {"name": "Sent", "flags": ["\\Sent"]},
+    {"name": "Drafts", "flags": ["\\Drafts"]},
+    {"name": "Trash", "flags": ["\\Trash"]},
+    {"name": "Archive", "flags": ["\\Archive"]},
+]
+
+
+@dataclass(frozen=True, slots=True)
+class MockDeliveryRecord:
+    id: str
+    kind: str
+    created_at: str
+    envelope_from: str | None
+    envelope_recipients: list[str]
+    subject: str | None
+    from_header: str | None
+    to_header: str | None
+    cc_header: str | None
+    bcc_header: str | None
+    message_id: str | None
+    size_bytes: int
+    body_preview: str | None
+    attachment_count: int
+    folder: str | None = None
+    smtp_host: str | None = None
+    imap_host: str | None = None
+    raw_filename: str | None = None
+    headers: dict[str, str] | None = None
+    attachments: list[dict[str, Any]] | None = None
+
+    def as_dict(self, *, include_raw: bool = False) -> dict[str, Any]:
+        data = {
+            "id": self.id,
+            "kind": self.kind,
+            "created_at": self.created_at,
+            "envelope_from": self.envelope_from,
+            "envelope_recipients": self.envelope_recipients,
+            "subject": self.subject,
+            "from_header": self.from_header,
+            "to_header": self.to_header,
+            "cc_header": self.cc_header,
+            "bcc_header": self.bcc_header,
+            "message_id": self.message_id,
+            "size_bytes": self.size_bytes,
+            "body_preview": self.body_preview,
+            "attachment_count": self.attachment_count,
+            "folder": self.folder,
+            "smtp_host": self.smtp_host,
+            "imap_host": self.imap_host,
+            "raw_filename": self.raw_filename,
+            "headers": self.headers or {},
+            "attachments": self.attachments or [],
+        }
+        if include_raw:
+            data["raw_eml"] = read_raw_message(self.id)
+        return data
+
+
+def _base_dir() -> Path:
+    path = Path(settings.mock_mailbox_dir)
+    path.mkdir(parents=True, exist_ok=True)
+    (path / "messages").mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _message_dir() -> Path:
+    return _base_dir() / "messages"
+
+
+def _failure_path() -> Path:
+    return _base_dir() / "failures.json"
+
+
+def normalize_mock_host(host: str | None) -> str:
+    return (host or "").strip().lower()
+
+
+def is_mock_smtp_host(host: str | None) -> bool:
+    return normalize_mock_host(host) in MOCK_SMTP_HOSTS
+
+
+def is_mock_imap_host(host: str | None) -> bool:
+    return normalize_mock_host(host) in MOCK_IMAP_HOSTS
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _message_to_bytes(message: EmailMessage | bytes) -> bytes:
+    if isinstance(message, bytes):
+        return message
+    return message.as_bytes(policy=policy.SMTP)
+
+
+def _parse_message(raw: bytes) -> EmailMessage:
+    return BytesParser(policy=policy.default).parsebytes(raw)
+
+
+def _header_text(message: EmailMessage, name: str) -> str | None:
+    value = message.get(name)
+    return str(value) if value is not None else None
+
+
+def _body_preview(message: EmailMessage) -> str | None:
+    body = None
+    try:
+        if message.is_multipart():
+            body = message.get_body(preferencelist=("plain", "html"))
+        else:
+            body = message
+        if body is None:
+            return None
+        content = body.get_content()
+    except Exception:
+        return None
+    text = re.sub(r"\s+", " ", str(content)).strip()
+    if not text:
+        return None
+    return text[:600]
+
+
+def _attachment_summaries(message: EmailMessage) -> list[dict[str, Any]]:
+    attachments: list[dict[str, Any]] = []
+    for part in message.iter_attachments():
+        payload = part.get_payload(decode=True) or b""
+        attachments.append(
+            {
+                "filename": part.get_filename(),
+                "content_type": part.get_content_type(),
+                "size_bytes": len(payload),
+            }
+        )
+    return attachments
+
+
+def _headers(message: EmailMessage) -> dict[str, str]:
+    return {str(key): str(value) for key, value in message.items()}
+
+
+def _record_from_raw(
+    raw: bytes,
+    *,
+    kind: str,
+    envelope_from: str | None = None,
+    envelope_recipients: list[str] | None = None,
+    folder: str | None = None,
+    smtp_host: str | None = None,
+    imap_host: str | None = None,
+) -> MockDeliveryRecord:
+    message_id = uuid4().hex
+    raw_filename = f"{message_id}.eml"
+    json_filename = f"{message_id}.json"
+    message = _parse_message(raw)
+    attachments = _attachment_summaries(message)
+    record = MockDeliveryRecord(
+        id=message_id,
+        kind=kind,
+        created_at=_now_iso(),
+        envelope_from=envelope_from,
+        envelope_recipients=list(envelope_recipients or []),
+        subject=_header_text(message, "Subject"),
+        from_header=_header_text(message, "From"),
+        to_header=_header_text(message, "To"),
+        cc_header=_header_text(message, "Cc"),
+        bcc_header=_header_text(message, "Bcc"),
+        message_id=_header_text(message, "Message-ID"),
+        size_bytes=len(raw),
+        body_preview=_body_preview(message),
+        attachment_count=len(attachments),
+        folder=folder,
+        smtp_host=smtp_host,
+        imap_host=imap_host,
+        raw_filename=raw_filename,
+        headers=_headers(message),
+        attachments=attachments,
+    )
+    message_dir = _message_dir()
+    (message_dir / raw_filename).write_bytes(raw)
+    (message_dir / json_filename).write_text(json.dumps(record.as_dict(), indent=2, ensure_ascii=False), encoding="utf-8")
+    return record
+
+
+def record_smtp_delivery(
+    message: EmailMessage,
+    *,
+    envelope_from: str,
+    envelope_recipients: list[str],
+    smtp_host: str | None = None,
+) -> MockDeliveryRecord:
+    return _record_from_raw(
+        _message_to_bytes(message),
+        kind="smtp",
+        envelope_from=envelope_from,
+        envelope_recipients=envelope_recipients,
+        smtp_host=smtp_host,
+    )
+
+
+def record_imap_append(
+    message_bytes: bytes,
+    *,
+    folder: str,
+    imap_host: str | None = None,
+) -> MockDeliveryRecord:
+    return _record_from_raw(message_bytes, kind="imap_append", folder=folder, imap_host=imap_host)
+
+
+def _load_record(record_id: str) -> dict[str, Any] | None:
+    path = _message_dir() / f"{record_id}.json"
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return None
+
+
+def list_records(*, kind: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    for path in _message_dir().glob("*.json"):
+        try:
+            record = json.loads(path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            continue
+        if kind and record.get("kind") != kind:
+            continue
+        records.append(record)
+    records.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True)
+    return records[: max(1, min(limit, 500))]
+
+
+def get_record(record_id: str, *, include_raw: bool = True) -> dict[str, Any] | None:
+    record = _load_record(record_id)
+    if record and include_raw:
+        record["raw_eml"] = read_raw_message(record_id)
+    return record
+
+
+def read_raw_message(record_id: str) -> str | None:
+    record = _load_record(record_id)
+    if not record:
+        return None
+    raw_filename = record.get("raw_filename")
+    if not raw_filename:
+        return None
+    raw_path = _message_dir() / str(raw_filename)
+    if not raw_path.exists():
+        return None
+    return raw_path.read_text(encoding="utf-8", errors="replace")
+
+
+def clear_records() -> int:
+    message_dir = _message_dir()
+    count = len(list(message_dir.glob("*.json")))
+    if message_dir.exists():
+        shutil.rmtree(message_dir)
+    message_dir.mkdir(parents=True, exist_ok=True)
+    return count
+
+
+def get_failures() -> dict[str, Any]:
+    path = _failure_path()
+    if not path.exists():
+        return {
+            "fail_next_smtp": False,
+            "fail_next_imap": False,
+            "smtp_reject_recipients_containing": None,
+        }
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        data = {}
+    return {
+        "fail_next_smtp": bool(data.get("fail_next_smtp")),
+        "fail_next_imap": bool(data.get("fail_next_imap")),
+        "smtp_reject_recipients_containing": data.get("smtp_reject_recipients_containing") or None,
+    }
+
+
+def set_failures(*, fail_next_smtp: bool | None = None, fail_next_imap: bool | None = None, smtp_reject_recipients_containing: str | None = None) -> dict[str, Any]:
+    current = get_failures()
+    if fail_next_smtp is not None:
+        current["fail_next_smtp"] = fail_next_smtp
+    if fail_next_imap is not None:
+        current["fail_next_imap"] = fail_next_imap
+    current["smtp_reject_recipients_containing"] = smtp_reject_recipients_containing or None
+    _failure_path().write_text(json.dumps(current, indent=2), encoding="utf-8")
+    return current
+
+
+def consume_fail_next_smtp() -> bool:
+    current = get_failures()
+    if current.get("fail_next_smtp"):
+        current["fail_next_smtp"] = False
+        _failure_path().write_text(json.dumps(current, indent=2), encoding="utf-8")
+        return True
+    return False
+
+
+def consume_fail_next_imap() -> bool:
+    current = get_failures()
+    if current.get("fail_next_imap"):
+        current["fail_next_imap"] = False
+        _failure_path().write_text(json.dumps(current, indent=2), encoding="utf-8")
+        return True
+    return False