inital commit

server/app/mailer/reports/__init__.py

@@ -0,0 +1 @@
+"""Reporting helpers for campaigns and jobs."""

server/app/mailer/reports/campaigns.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+import csv
+import io
+import math
+from collections import Counter
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.db.models import (
+    Campaign,
+    CampaignIssue,
+    CampaignJob,
+    CampaignVersion,
+    ImapAppendAttempt,
+    SendAttempt,
+)
+from app.mailer.campaign.loader import load_campaign_config
+from app.mailer.persistence.campaigns import _write_campaign_snapshot
+
+
+class CampaignReportError(RuntimeError):
+    pass
+
+
+def _utcnow_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _counter(values: list[str | None]) -> dict[str, int]:
+    return dict(Counter(value or "unknown" for value in values))
+
+
+def _get_campaign(session: Session, *, tenant_id: str, campaign_id: str) -> Campaign:
+    campaign = session.query(Campaign).filter(Campaign.tenant_id == tenant_id, Campaign.id == campaign_id).one_or_none()
+    if not campaign:
+        raise CampaignReportError(f"Campaign not found or not accessible: {campaign_id}")
+    return campaign
+
+
+def _current_version(session: Session, campaign: Campaign) -> CampaignVersion | None:
+    if not campaign.current_version_id:
+        return None
+    version = session.get(CampaignVersion, campaign.current_version_id)
+    if version and version.campaign_id == campaign.id:
+        return version
+    return None
+
+
+def _version_info(version: CampaignVersion | None) -> dict[str, Any] | None:
+    if not version:
+        return None
+    return {
+        "id": version.id,
+        "version_number": version.version_number,
+        "schema_version": version.schema_version,
+        "source_filename": version.source_filename,
+        "created_at": version.created_at.isoformat() if version.created_at else None,
+        "validation_summary": version.validation_summary,
+        "build_summary": version.build_summary,
+    }
+
+
+def _load_delivery_info(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, Any]:
+    """Extract rate-limit and IMAP settings from the version JSON where possible.
+
+    This stays best-effort so reports still work if the schema evolves or a
+    partial/invalid campaign snapshot exists.
+    """
+
+    default = {
+        "rate_limit": {"messages_per_minute": None, "concurrency": None},
+        "imap_append_sent": {"enabled": None, "folder": None},
+        "retry": {"max_attempts": None, "backoff_seconds": []},
+        "estimated_remaining_send_seconds": None,
+        "estimated_remaining_send_human": None,
+    }
+    if not version:
+        return default
+    try:
+        snapshot_path = _write_campaign_snapshot(version)
+        config = load_campaign_config(snapshot_path)
+    except Exception as exc:  # pragma: no cover - reporting should not fail hard here
+        default["load_error"] = str(exc)
+        return default
+
+    messages_per_minute = config.delivery.rate_limit.messages_per_minute
+    pending = [job for job in jobs if job.send_status in {"queued", "failed_temporary", "sending"}]
+    estimated_seconds = None
+    if messages_per_minute and pending:
+        estimated_seconds = int(math.ceil((len(pending) / messages_per_minute) * 60))
+
+    return {
+        "rate_limit": {
+            "messages_per_minute": messages_per_minute,
+            "concurrency": config.delivery.rate_limit.concurrency,
+        },
+        "imap_append_sent": {
+            "enabled": config.delivery.imap_append_sent.enabled,
+            "folder": config.delivery.imap_append_sent.folder,
+        },
+        "retry": {
+            "max_attempts": config.delivery.retry.max_attempts,
+            "backoff_seconds": config.delivery.retry.backoff_seconds,
+        },
+        "estimated_remaining_send_seconds": estimated_seconds,
+        "estimated_remaining_send_human": _human_duration(estimated_seconds),
+    }
+
+
+def _human_duration(seconds: int | None) -> str | None:
+    if seconds is None:
+        return None
+    if seconds < 60:
+        return f"{seconds}s"
+    minutes, sec = divmod(seconds, 60)
+    if minutes < 60:
+        return f"{minutes}m {sec}s" if sec else f"{minutes}m"
+    hours, minute = divmod(minutes, 60)
+    return f"{hours}h {minute}m" if minute else f"{hours}h"
+
+
+def _issue_summary_from_jobs(jobs: list[CampaignJob]) -> dict[str, Any]:
+    severity_counter: Counter[str] = Counter()
+    code_counter: Counter[str] = Counter()
+    behavior_counter: Counter[str] = Counter()
+    total = 0
+    for job in jobs:
+        for issue in job.issues_snapshot or []:
+            if not isinstance(issue, dict):
+                continue
+            total += 1
+            severity_counter[issue.get("severity") or "unknown"] += 1
+            code_counter[issue.get("code") or "unknown"] += 1
+            if issue.get("behavior"):
+                behavior_counter[issue["behavior"]] += 1
+    return {
+        "total": total,
+        "by_severity": dict(severity_counter),
+        "by_code": dict(code_counter),
+        "by_behavior": dict(behavior_counter),
+    }
+
+
+def _attachment_summary(jobs: list[CampaignJob]) -> dict[str, Any]:
+    status_counter: Counter[str] = Counter()
+    behavior_counter: Counter[str] = Counter()
+    total_configs = 0
+    total_matched_files = 0
+    zip_enabled = 0
+    missing = 0
+    ambiguous = 0
+    for job in jobs:
+        for attachment in job.resolved_attachments or []:
+            if not isinstance(attachment, dict):
+                continue
+            total_configs += 1
+            status = attachment.get("status") or "unknown"
+            status_counter[status] += 1
+            if attachment.get("behavior"):
+                behavior_counter[attachment["behavior"]] += 1
+            matches = attachment.get("matches") or []
+            if isinstance(matches, list):
+                total_matched_files += len(matches)
+            if attachment.get("zip_enabled"):
+                zip_enabled += 1
+            if status == "missing":
+                missing += 1
+            if status == "ambiguous":
+                ambiguous += 1
+    return {
+        "total_attachment_configs": total_configs,
+        "total_matched_files": total_matched_files,
+        "zip_enabled_configs": zip_enabled,
+        "missing_configs": missing,
+        "ambiguous_configs": ambiguous,
+        "by_status": dict(status_counter),
+        "by_behavior": dict(behavior_counter),
+    }
+
+
+def _recent_failures(jobs: list[CampaignJob], *, limit: int = 20) -> list[dict[str, Any]]:
+    failed = [job for job in jobs if job.last_error or str(job.send_status).startswith("failed") or job.imap_status == "failed"]
+    failed.sort(key=lambda job: job.updated_at or job.created_at, reverse=True)
+    return [
+        {
+            "job_id": job.id,
+            "entry_index": job.entry_index,
+            "entry_id": job.entry_id,
+            "recipient_email": job.recipient_email,
+            "validation_status": job.validation_status,
+            "send_status": job.send_status,
+            "imap_status": job.imap_status,
+            "attempt_count": job.attempt_count,
+            "last_error": job.last_error,
+            "updated_at": job.updated_at.isoformat() if job.updated_at else None,
+        }
+        for job in failed[:limit]
+    ]
+
+
+def _job_row(job: CampaignJob) -> dict[str, Any]:
+    return {
+        "job_id": job.id,
+        "entry_index": job.entry_index,
+        "entry_id": job.entry_id,
+        "recipient_email": job.recipient_email,
+        "subject": job.subject,
+        "build_status": job.build_status,
+        "validation_status": job.validation_status,
+        "queue_status": job.queue_status,
+        "send_status": job.send_status,
+        "imap_status": job.imap_status,
+        "attempt_count": job.attempt_count,
+        "queued_at": job.queued_at.isoformat() if job.queued_at else None,
+        "sent_at": job.sent_at.isoformat() if job.sent_at else None,
+        "last_error": job.last_error,
+        "eml_size_bytes": job.eml_size_bytes,
+        "issues_count": len(job.issues_snapshot or []),
+        "attachment_config_count": len(job.resolved_attachments or []),
+        "matched_file_count": sum(len(item.get("matches") or []) for item in (job.resolved_attachments or []) if isinstance(item, dict)),
+    }
+
+
+def generate_campaign_report(
+    session: Session,
+    *,
+    tenant_id: str,
+    campaign_id: str,
+    include_jobs: bool = False,
+    include_recent_failures: bool = True,
+) -> dict[str, Any]:
+    """Generate a dashboard/report payload for one campaign.
+
+    The shape is intentionally web-UI friendly: status counters for cards,
+    issue/attachment summaries for review panels, and optional job rows for
+    tables/export.
+    """
+
+    campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
+    version = _current_version(session, campaign)
+    jobs = (
+        session.query(CampaignJob)
+        .filter(CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id)
+        .order_by(CampaignJob.entry_index.asc())
+        .all()
+    )
+    job_ids = [job.id for job in jobs]
+    send_attempts = session.query(SendAttempt).filter(SendAttempt.job_id.in_(job_ids)).count() if job_ids else 0
+    imap_attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id.in_(job_ids)).count() if job_ids else 0
+    persisted_issues = session.query(CampaignIssue).filter(CampaignIssue.tenant_id == tenant_id, CampaignIssue.campaign_id == campaign.id).count()
+
+    validation_counts = _counter([job.validation_status for job in jobs])
+    queue_counts = _counter([job.queue_status for job in jobs])
+    send_counts = _counter([job.send_status for job in jobs])
+    imap_counts = _counter([job.imap_status for job in jobs])
+    build_counts = _counter([job.build_status for job in jobs])
+
+    queueable = sum(1 for job in jobs if job.validation_status in {"ready", "warning"} and job.build_status == "built")
+    needs_attention = sum(
+        1
+        for job in jobs
+        if job.validation_status in {"needs_review", "blocked"}
+        or job.send_status in {"failed_temporary", "failed_permanent"}
+        or job.imap_status == "failed"
+    )
+    sent = send_counts.get("sent", 0)
+    failed = send_counts.get("failed_temporary", 0) + send_counts.get("failed_permanent", 0)
+
+    report: dict[str, Any] = {
+        "generated_at": _utcnow_iso(),
+        "campaign": {
+            "id": campaign.id,
+            "external_id": campaign.external_id,
+            "name": campaign.name,
+            "description": campaign.description,
+            "status": campaign.status,
+            "created_at": campaign.created_at.isoformat() if campaign.created_at else None,
+            "updated_at": campaign.updated_at.isoformat() if campaign.updated_at else None,
+        },
+        "current_version": _version_info(version),
+        "cards": {
+            "jobs_total": len(jobs),
+            "queueable": queueable,
+            "needs_attention": needs_attention,
+            "sent": sent,
+            "failed": failed,
+            "imap_appended": imap_counts.get("appended", 0),
+            "imap_failed": imap_counts.get("failed", 0),
+        },
+        "status_counts": {
+            "build": build_counts,
+            "validation": validation_counts,
+            "queue": queue_counts,
+            "send": send_counts,
+            "imap": imap_counts,
+        },
+        "issues": {
+            **_issue_summary_from_jobs(jobs),
+            "persisted_campaign_issue_count": persisted_issues,
+        },
+        "attachments": _attachment_summary(jobs),
+        "attempts": {
+            "send_attempts": int(send_attempts),
+            "imap_append_attempts": int(imap_attempts),
+        },
+        "delivery": _load_delivery_info(version, jobs),
+    }
+    if include_recent_failures:
+        report["recent_failures"] = _recent_failures(jobs)
+    if include_jobs:
+        report["jobs"] = [_job_row(job) for job in jobs]
+    return report
+
+
+def generate_jobs_csv(session: Session, *, tenant_id: str, campaign_id: str) -> str:
+    campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
+    jobs = (
+        session.query(CampaignJob)
+        .filter(CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id)
+        .order_by(CampaignJob.entry_index.asc())
+        .all()
+    )
+    rows = [_job_row(job) for job in jobs]
+    fieldnames = [
+        "job_id",
+        "entry_index",
+        "entry_id",
+        "recipient_email",
+        "subject",
+        "build_status",
+        "validation_status",
+        "queue_status",
+        "send_status",
+        "imap_status",
+        "attempt_count",
+        "queued_at",
+        "sent_at",
+        "last_error",
+        "eml_size_bytes",
+        "issues_count",
+        "attachment_config_count",
+        "matched_file_count",
+    ]
+    buffer = io.StringIO()
+    writer = csv.DictWriter(buffer, fieldnames=fieldnames)
+    writer.writeheader()
+    writer.writerows(rows)
+    return buffer.getvalue()

server/app/mailer/reports/emailing.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from email.message import EmailMessage
+from email.utils import formataddr
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from app.db.models import Campaign, CampaignVersion
+from app.mailer.campaign.loader import load_campaign_config
+from app.mailer.campaign.models import CampaignConfig, SmtpConfig
+from app.mailer.persistence.campaigns import _write_campaign_snapshot
+from app.mailer.reports.campaigns import CampaignReportError, generate_campaign_report, generate_jobs_csv
+from app.mailer.sending.smtp import SmtpConfigurationError, SmtpSendResult, send_email_message
+
+
+class CampaignReportEmailError(RuntimeError):
+    pass
+
+
+@dataclass(slots=True)
+class CampaignReportEmailResult:
+    campaign_id: str
+    to: list[str]
+    subject: str
+    dry_run: bool
+    sent: bool
+    attached_jobs_csv: bool
+    attached_report_json: bool
+    smtp_host: str | None = None
+    smtp_port: int | None = None
+    accepted_count: int | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "campaign_id": self.campaign_id,
+            "to": self.to,
+            "subject": self.subject,
+            "dry_run": self.dry_run,
+            "sent": self.sent,
+            "attached_jobs_csv": self.attached_jobs_csv,
+            "attached_report_json": self.attached_report_json,
+            "smtp_host": self.smtp_host,
+            "smtp_port": self.smtp_port,
+            "accepted_count": self.accepted_count,
+        }
+
+
+def _current_version(session: Session, campaign: Campaign) -> CampaignVersion:
+    version = session.get(CampaignVersion, campaign.current_version_id) if campaign.current_version_id else None
+    if version is None:
+        version = (
+            session.query(CampaignVersion)
+            .filter(CampaignVersion.campaign_id == campaign.id)
+            .order_by(CampaignVersion.version_number.desc())
+            .first()
+        )
+    if version is None:
+        raise CampaignReportEmailError("Campaign has no version")
+    return version
+
+
+def _load_config(version: CampaignVersion) -> CampaignConfig:
+    snapshot_path = _write_campaign_snapshot(version)
+    return load_campaign_config(snapshot_path)
+
+
+def _effective_from(config: CampaignConfig) -> tuple[str, str | None]:
+    if config.recipients.from_:
+        return config.recipients.from_.email, config.recipients.from_.name
+    if config.server.smtp and config.server.smtp.username and "@" in config.server.smtp.username:
+        return config.server.smtp.username, None
+    raise SmtpConfigurationError("Report email requires recipients.from.email or an SMTP username that is an email address")
+
+
+def _text_summary(report: dict[str, Any]) -> str:
+    campaign = report["campaign"]
+    cards = report["cards"]
+    status = report["status_counts"]
+    delivery = report.get("delivery", {})
+    lines = [
+        f"Campaign report: {campaign['name']}",
+        "",
+        f"Campaign ID: {campaign['id']}",
+        f"External ID: {campaign['external_id']}",
+        f"Status: {campaign['status']}",
+        "",
+        "Overview",
+        f"- Jobs total: {cards['jobs_total']}",
+        f"- Queueable: {cards['queueable']}",
+        f"- Needs attention: {cards['needs_attention']}",
+        f"- Sent: {cards['sent']}",
+        f"- Failed: {cards['failed']}",
+        f"- IMAP appended: {cards['imap_appended']}",
+        f"- IMAP failed: {cards['imap_failed']}",
+        "",
+        f"Build status: {status.get('build', {})}",
+        f"Validation status: {status.get('validation', {})}",
+        f"Queue status: {status.get('queue', {})}",
+        f"Send status: {status.get('send', {})}",
+        f"IMAP status: {status.get('imap', {})}",
+    ]
+    if delivery.get("estimated_remaining_send_human"):
+        lines.extend(["", f"Estimated remaining send time: {delivery['estimated_remaining_send_human']}"])
+    lines.extend(["", "This report was generated by MultiMailer."])
+    return "\n".join(lines)
+
+
+def build_report_message(
+    *,
+    campaign: Campaign,
+    config: CampaignConfig,
+    report: dict[str, Any],
+    to: list[str],
+    jobs_csv: str | None = None,
+    report_json: dict[str, Any] | None = None,
+) -> EmailMessage:
+    from_email, from_name = _effective_from(config)
+    subject = f"MultiMailer report: {campaign.name}"
+
+    msg = EmailMessage()
+    msg["Subject"] = subject
+    msg["From"] = formataddr((from_name or from_email, from_email))
+    msg["To"] = ", ".join(to)
+    msg["X-MultiMailer-Report"] = "campaign"
+    msg.set_content(_text_summary(report))
+
+    if jobs_csv is not None:
+        filename = f"multimailer-{campaign.external_id}-jobs.csv"
+        msg.add_attachment(jobs_csv.encode("utf-8"), maintype="text", subtype="csv", filename=filename)
+    if report_json is not None:
+        filename = f"multimailer-{campaign.external_id}-report.json"
+        msg.add_attachment(
+            json.dumps(report_json, indent=2, ensure_ascii=False, default=str).encode("utf-8"),
+            maintype="application",
+            subtype="json",
+            filename=filename,
+        )
+    return msg
+
+
+def send_campaign_report_email(
+    session: Session,
+    *,
+    tenant_id: str,
+    campaign_id: str,
+    to: list[str],
+    include_jobs: bool = False,
+    attach_jobs_csv: bool = True,
+    attach_report_json: bool = False,
+    dry_run: bool = False,
+) -> CampaignReportEmailResult:
+    campaign = session.get(Campaign, campaign_id)
+    if not campaign or campaign.tenant_id != tenant_id:
+        raise CampaignReportError("Campaign not found")
+    if not to:
+        raise CampaignReportEmailError("At least one report recipient is required")
+
+    version = _current_version(session, campaign)
+    config = _load_config(version)
+    smtp_config: SmtpConfig | None = config.server.smtp
+    if smtp_config is None:
+        raise SmtpConfigurationError("Campaign has no SMTP configuration")
+
+    report = generate_campaign_report(session, tenant_id=tenant_id, campaign_id=campaign_id, include_jobs=include_jobs)
+    jobs_csv = generate_jobs_csv(session, tenant_id=tenant_id, campaign_id=campaign_id) if attach_jobs_csv else None
+    report_json = report if attach_report_json else None
+    message = build_report_message(
+        campaign=campaign,
+        config=config,
+        report=report,
+        to=to,
+        jobs_csv=jobs_csv,
+        report_json=report_json,
+    )
+    envelope_from, _ = _effective_from(config)
+
+    if dry_run:
+        return CampaignReportEmailResult(
+            campaign_id=campaign.id,
+            to=to,
+            subject=str(message["Subject"]),
+            dry_run=True,
+            sent=False,
+            attached_jobs_csv=jobs_csv is not None,
+            attached_report_json=report_json is not None,
+            smtp_host=smtp_config.host,
+            smtp_port=smtp_config.port,
+        )
+
+    result: SmtpSendResult = send_email_message(
+        message,
+        smtp_config=smtp_config,
+        envelope_from=envelope_from,
+        envelope_recipients=to,
+    )
+    return CampaignReportEmailResult(
+        campaign_id=campaign.id,
+        to=to,
+        subject=str(message["Subject"]),
+        dry_run=False,
+        sent=True,
+        attached_jobs_csv=jobs_csv is not None,
+        attached_report_json=report_json is not None,
+        smtp_host=result.host,
+        smtp_port=result.port,
+        accepted_count=result.accepted_count,
+    )