from __future__ import annotations import csv import io import logging import math from collections import Counter from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any from sqlalchemy.orm import Session from govoplan_core.settings import settings as core_settings from govoplan_campaign.backend.db.models import ( Campaign, CampaignIssue, CampaignJob, CampaignVersion, ImapAppendAttempt, PostboxDeliveryAttempt, SendAttempt, ) from govoplan_campaign.backend.sending.execution import ExecutionSnapshot from govoplan_campaign.backend.response_security import ( public_campaign_payload, public_delivery_result_message, public_source_filename, ) class CampaignReportError(RuntimeError): pass logger = logging.getLogger(__name__) CAMPAIGN_JSON_JOB_LIMIT = 5_000 CAMPAIGN_CSV_JOB_LIMIT = 100_000 def _utcnow_iso() -> str: return datetime.now(timezone.utc).isoformat() def _counter(values: list[str | None]) -> dict[str, int]: return dict(Counter(value or "unknown" for value in values)) def _get_campaign(session: Session, *, tenant_id: str, campaign_id: str) -> Campaign: campaign = session.query(Campaign).filter(Campaign.tenant_id == tenant_id, Campaign.id == campaign_id).one_or_none() if not campaign: raise CampaignReportError(f"Campaign not found or not accessible: {campaign_id}") return campaign def _selected_version( session: Session, campaign: Campaign, version_id: str | None = None, ) -> CampaignVersion | None: wanted = version_id or campaign.current_version_id if not wanted: return None version = session.get(CampaignVersion, wanted) if not version or version.campaign_id != campaign.id: raise CampaignReportError(f"Campaign version not found or not part of campaign: {wanted}") return version def _version_info( version: CampaignVersion | None, *, include_diagnostics: bool = False, ) -> dict[str, Any] | None: if not version: return None return { "id": version.id, "version_number": version.version_number, "schema_version": version.schema_version, "source_filename": public_source_filename(version.source_filename), "created_at": version.created_at.isoformat() if version.created_at else None, "validation_summary": public_campaign_payload( version.validation_summary, include_diagnostics=include_diagnostics, ), "build_summary": public_campaign_payload( version.build_summary, include_diagnostics=include_diagnostics, ), "execution_snapshot_hash": version.execution_snapshot_hash, "execution_snapshot_at": version.execution_snapshot_at.isoformat() if version.execution_snapshot_at else None, "delivery_mode": version.delivery_mode, "delivery_mode_selected_at": ( version.delivery_mode_selected_at.isoformat() if version.delivery_mode_selected_at else None ), } def _load_delivery_info( version: CampaignVersion | None, jobs: list[CampaignJob] | None = None, *, pending_job_count: int | None = None, include_diagnostics: bool = False, ) -> dict[str, Any]: """Read deterministic delivery settings from the immutable execution snapshot.""" default = { "rate_limit": {"messages_per_minute": None, "concurrency": None}, "imap_append_sent": {"enabled": None, "folder": None}, "retry": {"max_attempts": None, "backoff_seconds": []}, "execution_snapshot_hash": None, "execution_snapshot_at": None, "snapshot_version": None, "built_at": None, "job_count": 0, "queueable_job_count": 0, "estimated_remaining_send_seconds": None, "estimated_remaining_send_human": None, "delivery_mode": getattr(version, "delivery_mode", None) if version else None, "delivery_mode_selected_at": ( getattr(version, "delivery_mode_selected_at", None).isoformat() if version and getattr(version, "delivery_mode_selected_at", None) else None ), } if include_diagnostics: default.update( { "build_token": None, # nosec B105 - absent report value, not a credential. "job_manifest_sha256": None, "effective_policy_sha256": None, "smtp_transport_revision": None, "imap_transport_revision": None, "background_workers_enabled": bool(core_settings.celery_enabled), } ) if not version or not isinstance(version.execution_snapshot, dict): default["load_error"] = "No execution snapshot exists; rebuild the current locked version before delivery." return default try: snapshot = ExecutionSnapshot.model_validate(version.execution_snapshot) except Exception: # pragma: no cover - reporting should remain available logger.warning("Campaign execution snapshot could not be loaded for a report") default["load_error"] = "Execution snapshot is invalid; rebuild the selected version before delivery." return default messages_per_minute = snapshot.delivery.rate_limit.messages_per_minute if pending_job_count is None: pending_job_count = sum( 1 for job in jobs or [] if job.send_status in {"queued", "claimed", "sending"} ) estimated_seconds = None if messages_per_minute and pending_job_count: estimated_seconds = int( math.ceil((pending_job_count / messages_per_minute) * 60) ) result = { "rate_limit": { "messages_per_minute": messages_per_minute, "concurrency": snapshot.delivery.rate_limit.concurrency, }, "imap_append_sent": { "enabled": snapshot.delivery.imap_append_sent.enabled, "folder": snapshot.delivery.imap_append_sent.folder, }, "retry": { "max_attempts": snapshot.delivery.retry.max_attempts, "backoff_seconds": snapshot.delivery.retry.backoff_seconds, }, "execution_snapshot_hash": version.execution_snapshot_hash, "execution_snapshot_at": version.execution_snapshot_at.isoformat() if version.execution_snapshot_at else None, "snapshot_version": snapshot.snapshot_version, "built_at": snapshot.built_at, "job_count": snapshot.job_count, "queueable_job_count": snapshot.queueable_job_count, "estimated_remaining_send_seconds": estimated_seconds, "estimated_remaining_send_human": _human_duration(estimated_seconds), "delivery_mode": version.delivery_mode, "delivery_mode_selected_at": ( version.delivery_mode_selected_at.isoformat() if version.delivery_mode_selected_at else None ), } if include_diagnostics: result.update( { "build_token": snapshot.build_token, "job_manifest_sha256": snapshot.job_manifest_sha256, "effective_policy_sha256": snapshot.effective_policy_sha256, "smtp_transport_revision": snapshot.smtp_transport_revision, "imap_transport_revision": snapshot.imap_transport_revision, "background_workers_enabled": bool(core_settings.celery_enabled), } ) return result def _human_duration(seconds: int | None) -> str | None: if seconds is None: return None if seconds < 60: return f"{seconds}s" minutes, sec = divmod(seconds, 60) if minutes < 60: return f"{minutes}m {sec}s" if sec else f"{minutes}m" hours, minute = divmod(minutes, 60) return f"{hours}h {minute}m" if minute else f"{hours}h" def _issue_summary_from_jobs(jobs: list[CampaignJob]) -> dict[str, Any]: severity_counter: Counter[str] = Counter() code_counter: Counter[str] = Counter() behavior_counter: Counter[str] = Counter() total = 0 for job in jobs: for issue in job.issues_snapshot or []: if not isinstance(issue, dict): continue total += 1 severity_counter[issue.get("severity") or "unknown"] += 1 code_counter[issue.get("code") or "unknown"] += 1 if issue.get("behavior"): behavior_counter[issue["behavior"]] += 1 return { "total": total, "by_severity": dict(severity_counter), "by_code": dict(code_counter), "by_behavior": dict(behavior_counter), } def _attachment_summary(jobs: list[CampaignJob]) -> dict[str, Any]: status_counter: Counter[str] = Counter() behavior_counter: Counter[str] = Counter() total_configs = 0 total_matched_files = 0 zip_enabled = 0 missing = 0 ambiguous = 0 for job in jobs: for attachment in job.resolved_attachments or []: if not isinstance(attachment, dict): continue total_configs += 1 status = attachment.get("status") or "unknown" status_counter[status] += 1 if attachment.get("behavior"): behavior_counter[attachment["behavior"]] += 1 matches = attachment.get("matches") or [] if isinstance(matches, list): total_matched_files += len(matches) if attachment.get("zip_enabled"): zip_enabled += 1 if status == "missing": missing += 1 if status == "ambiguous": ambiguous += 1 return { "total_attachment_configs": total_configs, "total_matched_files": total_matched_files, "zip_enabled_configs": zip_enabled, "missing_configs": missing, "ambiguous_configs": ambiguous, "by_status": dict(status_counter), "by_behavior": dict(behavior_counter), } @dataclass(slots=True) class _JobReportAggregate: total: int = 0 pending: int = 0 queueable: int = 0 queueable_unattempted: int = 0 retryable: int = 0 cancellable: int = 0 needs_attention: int = 0 build: Counter[str] = field(default_factory=Counter) validation: Counter[str] = field(default_factory=Counter) queue: Counter[str] = field(default_factory=Counter) send: Counter[str] = field(default_factory=Counter) postbox: Counter[str] = field(default_factory=Counter) imap: Counter[str] = field(default_factory=Counter) issue_total: int = 0 issue_severity: Counter[str] = field(default_factory=Counter) issue_code: Counter[str] = field(default_factory=Counter) issue_behavior: Counter[str] = field(default_factory=Counter) attachment_total: int = 0 attachment_matches: int = 0 attachment_zip_enabled: int = 0 attachment_missing: int = 0 attachment_ambiguous: int = 0 attachment_status: Counter[str] = field(default_factory=Counter) attachment_behavior: Counter[str] = field(default_factory=Counter) recent_failures: list[CampaignJob] = field(default_factory=list) def add( self, job: CampaignJob, *, retry_max_attempts: int | None, include_recent_failures: bool, ) -> None: self.total += 1 self.build[job.build_status or "unknown"] += 1 self.validation[job.validation_status or "unknown"] += 1 self.queue[job.queue_status or "unknown"] += 1 self.send[job.send_status or "unknown"] += 1 self.postbox[job.postbox_status or "unknown"] += 1 self.imap[job.imap_status or "unknown"] += 1 if job.send_status in {"queued", "claimed", "sending"}: self.pending += 1 if ( job.validation_status in {"ready", "warning"} and job.build_status == "built" ): self.queueable += 1 if ( job.attempt_count == 0 and job.postbox_attempt_count == 0 and job.send_status in {"not_queued", "cancelled"} and job.validation_status in {"ready", "warning"} and job.build_status == "built" ): self.queueable_unattempted += 1 if ( job.send_status in {"failed_temporary", "partially_accepted"} and ( retry_max_attempts is None or job.attempt_count < retry_max_attempts ) ): self.retryable += 1 if job.send_status not in { "skipped", "smtp_accepted", "postbox_accepted", "delivered", "partially_accepted", "sent", "outcome_unknown", "claimed", "sending", "cancelled", }: self.cancellable += 1 if _job_needs_attention(job): self.needs_attention += 1 self._add_issues(job) self._add_attachments(job) if include_recent_failures and _job_is_recent_failure(job): self.recent_failures.append(job) self.recent_failures.sort( key=lambda item: item.updated_at or item.created_at, reverse=True, ) del self.recent_failures[20:] def _add_issues(self, job: CampaignJob) -> None: for issue in job.issues_snapshot or []: if not isinstance(issue, dict): continue self.issue_total += 1 self.issue_severity[issue.get("severity") or "unknown"] += 1 self.issue_code[issue.get("code") or "unknown"] += 1 if issue.get("behavior"): self.issue_behavior[str(issue["behavior"])] += 1 def _add_attachments(self, job: CampaignJob) -> None: for attachment in job.resolved_attachments or []: if not isinstance(attachment, dict): continue self.attachment_total += 1 status_value = str(attachment.get("status") or "unknown") self.attachment_status[status_value] += 1 if attachment.get("behavior"): self.attachment_behavior[str(attachment["behavior"])] += 1 matches = attachment.get("matches") if isinstance(matches, list): self.attachment_matches += len(matches) if attachment.get("zip_enabled"): self.attachment_zip_enabled += 1 if status_value == "missing": self.attachment_missing += 1 if status_value == "ambiguous": self.attachment_ambiguous += 1 def _job_needs_attention(job: CampaignJob) -> bool: return ( job.validation_status in {"needs_review", "blocked"} or job.send_status in { "failed_temporary", "failed_permanent", "partially_accepted", "outcome_unknown", "claimed", "sending", } or job.postbox_status in { "partially_accepted", "rejected_temporary", "rejected_permanent", "outcome_unknown", } or job.imap_status == "failed" ) def _job_is_recent_failure(job: CampaignJob) -> bool: return bool( job.last_error or str(job.send_status).startswith("failed") or job.send_status == "partially_accepted" or job.postbox_status in { "partially_accepted", "rejected_temporary", "rejected_permanent", "outcome_unknown", } or job.imap_status == "failed" ) def _recent_failures(jobs: list[CampaignJob], *, limit: int = 20) -> list[dict[str, Any]]: failed = [job for job in jobs if _job_is_recent_failure(job)] failed.sort(key=lambda job: job.updated_at or job.created_at, reverse=True) return [ { "job_id": job.id, "entry_index": job.entry_index, "entry_id": job.entry_id, "recipient_email": job.recipient_email, "validation_status": job.validation_status, "send_status": job.send_status, "postbox_status": job.postbox_status, "imap_status": job.imap_status, "attempt_count": job.attempt_count, "postbox_attempt_count": job.postbox_attempt_count, "last_error": public_delivery_result_message( last_error=job.last_error, send_status=job.send_status, imap_status=job.imap_status, postbox_status=job.postbox_status, ), "updated_at": job.updated_at.isoformat() if job.updated_at else None, } for job in failed[:limit] ] def _job_row(job: CampaignJob, *, include_diagnostics: bool = False) -> dict[str, Any]: row = { "job_id": job.id, "entry_index": job.entry_index, "entry_id": job.entry_id, "recipient_email": job.recipient_email, "subject": job.subject, "build_status": job.build_status, "validation_status": job.validation_status, "queue_status": job.queue_status, "send_status": job.send_status, "delivery_channel_policy": getattr( job, "delivery_channel_policy", "mail", ), "postbox_status": getattr( job, "postbox_status", "not_requested", ), "imap_status": job.imap_status, "attempt_count": job.attempt_count, "postbox_attempt_count": getattr(job, "postbox_attempt_count", 0), "postbox_target_count": len( getattr(job, "resolved_postbox_targets", None) or [] ), "queued_at": job.queued_at.isoformat() if job.queued_at else None, "outcome_unknown_at": job.outcome_unknown_at.isoformat() if job.outcome_unknown_at else None, "sent_at": job.sent_at.isoformat() if job.sent_at else None, "last_error": public_delivery_result_message( last_error=job.last_error, send_status=job.send_status, imap_status=job.imap_status, postbox_status=getattr(job, "postbox_status", "not_requested"), ), "eml_size_bytes": job.eml_size_bytes, "eml_sha256": job.eml_sha256, "issues_count": len(job.issues_snapshot or []), "attachment_config_count": len(job.resolved_attachments or []), "matched_file_count": sum(len(item.get("matches") or []) for item in (job.resolved_attachments or []) if isinstance(item, dict)), } if include_diagnostics: row.update( { "claimed_at": job.claimed_at.isoformat() if job.claimed_at else None, "smtp_started_at": job.smtp_started_at.isoformat() if job.smtp_started_at else None, } ) return row def _address_summary(value: Any) -> str: if value is None: return "" if isinstance(value, str): return value if isinstance(value, dict): email = str(value.get("email") or "").strip() name = str(value.get("name") or "").strip() if name and email: return f"{name} <{email}>" return email or name if isinstance(value, list): return "; ".join(item for item in (_address_summary(item) for item in value) if item) return str(value) def _latest_by_job_id(attempts: list[Any]) -> dict[str, Any]: latest: dict[str, Any] = {} for attempt in attempts: current = latest.get(attempt.job_id) if current is None or (attempt.attempt_number or 0) >= (current.attempt_number or 0): latest[attempt.job_id] = attempt return latest def _attachment_names(attachments: list[dict[str, Any]] | None) -> str: names: list[str] = [] for attachment in attachments or []: if not isinstance(attachment, dict): continue for key in ("message_filenames", "zip_entry_names"): values = attachment.get(key) if isinstance(values, list): names.extend(str(value) for value in values if value) zip_filename = attachment.get("zip_filename") if zip_filename: names.append(str(zip_filename)) matches = attachment.get("matches") if isinstance(matches, list): for match in matches: if isinstance(match, dict): filename = match.get("filename") or match.get("name") or match.get("display_name") if filename: names.append(str(filename)) elif match: names.append(str(match).rsplit("/", 1)[-1]) seen: set[str] = set() deduplicated = [] for name in names: if name and name not in seen: seen.add(name) deduplicated.append(name) return "; ".join(deduplicated) def _job_evidence_row( job: CampaignJob, *, latest_smtp: SendAttempt | None = None, latest_imap: ImapAppendAttempt | None = None, include_diagnostics: bool = False, ) -> dict[str, Any]: row = _job_row(job, include_diagnostics=include_diagnostics) recipients = job.resolved_recipients or {} row.update({ "campaign_id": job.campaign_id, "campaign_version_id": job.campaign_version_id, "message_id_header": job.message_id_header, "from": _address_summary(recipients.get("from")), "to": _address_summary(recipients.get("to")), "cc": _address_summary(recipients.get("cc")), "bcc": _address_summary(recipients.get("bcc")), "reply_to": _address_summary(recipients.get("reply_to")), "postbox_targets": "; ".join( str(target.get("address") or target.get("name") or target.get("postbox_id") or "") for target in ( getattr(job, "resolved_postbox_targets", None) or [] ) if isinstance(target, dict) ), "attachment_names": _attachment_names(job.resolved_attachments), "latest_smtp_attempt_number": latest_smtp.attempt_number if latest_smtp else None, "latest_smtp_status": latest_smtp.status if latest_smtp else None, "latest_smtp_status_code": latest_smtp.smtp_status_code if latest_smtp else None, "latest_smtp_started_at": latest_smtp.started_at.isoformat() if latest_smtp and latest_smtp.started_at else None, "latest_smtp_finished_at": latest_smtp.finished_at.isoformat() if latest_smtp and latest_smtp.finished_at else None, "latest_imap_attempt_number": latest_imap.attempt_number if latest_imap else None, "latest_imap_status": latest_imap.status if latest_imap else None, "latest_imap_folder": latest_imap.folder if latest_imap else None, "latest_imap_created_at": latest_imap.created_at.isoformat() if latest_imap and latest_imap.created_at else None, "latest_imap_updated_at": latest_imap.updated_at.isoformat() if latest_imap and latest_imap.updated_at else None, }) return row def generate_campaign_report( session: Session, *, tenant_id: str, campaign_id: str, version_id: str | None = None, include_jobs: bool = False, include_recent_failures: bool = False, include_diagnostics: bool = False, ) -> dict[str, Any]: """Generate a dashboard/report payload for one campaign. The shape is intentionally web-UI friendly: status counters for cards, issue/attachment summaries for review panels, and optional job rows for tables/export. """ campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id) version = _selected_version(session, campaign, version_id) jobs = _report_jobs( session, tenant_id=tenant_id, campaign_id=campaign.id, version=version, ) aggregate = _JobReportAggregate() job_rows: list[dict[str, Any]] = [] retry_max_attempts = _retry_max_attempts(version) iterator = jobs.yield_per(500) if hasattr(jobs, "yield_per") else jobs for job in iterator: aggregate.add( job, retry_max_attempts=retry_max_attempts, include_recent_failures=include_recent_failures, ) if include_jobs: if len(job_rows) >= CAMPAIGN_JSON_JOB_LIMIT: raise CampaignReportError( "The recipient-level JSON report exceeds the safe row " f"limit of {CAMPAIGN_JSON_JOB_LIMIT}; use the CSV export " "or the paginated recipient table." ) job_rows.append( _job_row(job, include_diagnostics=include_diagnostics) ) report = _campaign_report_payload( session, campaign=campaign, version=version, aggregate=aggregate, tenant_id=tenant_id, include_recent_failures=include_recent_failures, include_diagnostics=include_diagnostics, ) if include_jobs: report["jobs"] = job_rows return report def _report_jobs( session: Session, *, tenant_id: str, campaign_id: str, version: CampaignVersion | None, ) -> Any: jobs_query = session.query(CampaignJob).filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign_id, ) if version: jobs_query = jobs_query.filter(CampaignJob.campaign_version_id == version.id) else: jobs_query = jobs_query.filter(False) return jobs_query.order_by(CampaignJob.entry_index.asc()) def _campaign_report_payload( session: Session, *, campaign: Campaign, version: CampaignVersion | None, aggregate: _JobReportAggregate, tenant_id: str, include_recent_failures: bool, include_diagnostics: bool, ) -> dict[str, Any]: report = { "generated_at": _utcnow_iso(), "campaign": _campaign_report_campaign_payload(campaign), "current_version": _version_info(version, include_diagnostics=include_diagnostics), "selected_version_id": version.id if version else None, "cards": _campaign_report_cards_from_aggregate(version, aggregate), "status_counts": _campaign_report_status_counts_from_aggregate( version, aggregate, ), "issues": { "total": aggregate.issue_total, "by_severity": dict(aggregate.issue_severity), "by_code": dict(aggregate.issue_code), "by_behavior": dict(aggregate.issue_behavior), "persisted_campaign_issue_count": _persisted_campaign_issue_count( session, tenant_id=tenant_id, campaign_id=campaign.id, version=version, ), }, "attachments": { "total_attachment_configs": aggregate.attachment_total, "total_matched_files": aggregate.attachment_matches, "zip_enabled_configs": aggregate.attachment_zip_enabled, "missing_configs": aggregate.attachment_missing, "ambiguous_configs": aggregate.attachment_ambiguous, "by_status": dict(aggregate.attachment_status), "by_behavior": dict(aggregate.attachment_behavior), }, "attempts": _campaign_report_attempt_counts( session, tenant_id=tenant_id, campaign_id=campaign.id, version=version, ), "delivery": _load_delivery_info( version, pending_job_count=aggregate.pending, include_diagnostics=include_diagnostics, ), } if include_recent_failures: report["recent_failures"] = _recent_failures( aggregate.recent_failures, ) return report def _campaign_report_campaign_payload(campaign: Campaign) -> dict[str, Any]: return { "id": campaign.id, "external_id": campaign.external_id, "name": campaign.name, "description": campaign.description, "status": campaign.status, "created_at": campaign.created_at.isoformat() if campaign.created_at else None, "updated_at": campaign.updated_at.isoformat() if campaign.updated_at else None, } def _campaign_report_attempt_counts( session: Session, *, tenant_id: str, campaign_id: str, version: CampaignVersion | None, ) -> dict[str, int]: def count(model: type[Any]) -> int: query = ( session.query(model) .join(CampaignJob, model.job_id == CampaignJob.id) .filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign_id, ) ) if version is not None: query = query.filter( CampaignJob.campaign_version_id == version.id, ) else: query = query.filter(False) return int(query.count()) return { "send_attempts": count(SendAttempt), "imap_append_attempts": count(ImapAppendAttempt), "postbox_delivery_attempts": count(PostboxDeliveryAttempt), } def _persisted_campaign_issue_count( session: Session, *, tenant_id: str, campaign_id: str, version: CampaignVersion | None, ) -> int: issue_query = session.query(CampaignIssue).filter( CampaignIssue.tenant_id == tenant_id, CampaignIssue.campaign_id == campaign_id, ) if version: issue_query = issue_query.filter(CampaignIssue.campaign_version_id == version.id) else: issue_query = issue_query.filter(False) return int(issue_query.count()) def _campaign_report_status_counts(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, dict[str, int]]: aggregate = _aggregate_job_list(version, jobs) return _campaign_report_status_counts_from_aggregate(version, aggregate) def _campaign_report_status_counts_from_aggregate( version: CampaignVersion | None, aggregate: _JobReportAggregate, ) -> dict[str, dict[str, int]]: validation_counts = dict(aggregate.validation) inactive_entries = _inactive_entry_count(version) if inactive_entries: validation_counts["inactive"] = inactive_entries return { "build": dict(aggregate.build), "validation": validation_counts, "queue": dict(aggregate.queue), "send": dict(aggregate.send), "postbox": dict(aggregate.postbox), "imap": dict(aggregate.imap), } def _campaign_report_cards_from_aggregate( version: CampaignVersion | None, aggregate: _JobReportAggregate, ) -> dict[str, object]: send_counts = aggregate.send imap_counts = aggregate.imap sent = ( send_counts.get("sent", 0) + send_counts.get("smtp_accepted", 0) + send_counts.get("postbox_accepted", 0) + send_counts.get("delivered", 0) + send_counts.get("partially_accepted", 0) ) failed = ( send_counts.get("failed_temporary", 0) + send_counts.get("failed_permanent", 0) ) outcome_unknown = send_counts.get("outcome_unknown", 0) not_attempted = send_counts.get("not_queued", 0) cancelled = send_counts.get("cancelled", 0) inactive_entries = _inactive_entry_count(version) return { "jobs_total": aggregate.total, "inactive": inactive_entries, "queueable": aggregate.queueable, "queueable_unattempted": aggregate.queueable_unattempted, "retryable": aggregate.retryable, "cancellable": aggregate.cancellable, "needs_attention": aggregate.needs_attention, "sent": sent, "smtp_accepted": ( send_counts.get("sent", 0) + send_counts.get("smtp_accepted", 0) ), "postbox_accepted": send_counts.get("postbox_accepted", 0), "delivered": send_counts.get("delivered", 0), "partially_accepted": send_counts.get("partially_accepted", 0), "failed": failed, "outcome_unknown": outcome_unknown, "not_attempted": not_attempted, "skipped": send_counts.get("skipped", 0), "queued_or_active": aggregate.pending, "cancelled": cancelled, "partially_completed": bool( send_counts.get("partially_accepted", 0) or sent and (failed or outcome_unknown or not_attempted or cancelled) ), "imap_appended": imap_counts.get("appended", 0), "imap_failed": imap_counts.get("failed", 0), "imap_skipped": imap_counts.get("skipped", 0), } def _campaign_report_cards(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, object]: return _campaign_report_cards_from_aggregate( version, _aggregate_job_list(version, jobs), ) def _aggregate_job_list( version: CampaignVersion | None, jobs: list[CampaignJob], ) -> _JobReportAggregate: aggregate = _JobReportAggregate() retry_max_attempts = _retry_max_attempts(version) for job in jobs: aggregate.add( job, retry_max_attempts=retry_max_attempts, include_recent_failures=False, ) return aggregate def _retry_max_attempts(version: CampaignVersion | None) -> int | None: if version is None or not isinstance( getattr(version, "execution_snapshot", None), dict, ): return None try: return ExecutionSnapshot.model_validate(version.execution_snapshot).delivery.retry.max_attempts except Exception: return None def _inactive_entry_count(version: CampaignVersion | None) -> int: build_summary = version.build_summary if version and isinstance(version.build_summary, dict) else {} return int(build_summary.get("inactive_count") or build_summary.get("inactive_entries_count") or 0) def generate_jobs_csv( session: Session, *, tenant_id: str, campaign_id: str, version_id: str | None = None, include_diagnostics: bool = False, ) -> str: campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id) version = _selected_version(session, campaign, version_id) jobs_query = session.query(CampaignJob).filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id, ) if version: jobs_query = jobs_query.filter(CampaignJob.campaign_version_id == version.id) else: jobs_query = jobs_query.filter(False) jobs = ( jobs_query .order_by(CampaignJob.entry_index.asc()) .limit(CAMPAIGN_CSV_JOB_LIMIT + 1) .all() ) if len(jobs) > CAMPAIGN_CSV_JOB_LIMIT: raise CampaignReportError( "The campaign CSV export exceeds the safe row limit of " f"{CAMPAIGN_CSV_JOB_LIMIT}. Split the export by recipient filter." ) job_ids = [job.id for job in jobs] latest_smtp = _latest_attempts_for_jobs( session, SendAttempt, job_ids, ) latest_imap = _latest_attempts_for_jobs( session, ImapAppendAttempt, job_ids, ) rows = [ _job_evidence_row( job, latest_smtp=latest_smtp.get(job.id), latest_imap=latest_imap.get(job.id), include_diagnostics=include_diagnostics, ) for job in jobs ] fieldnames = [ "job_id", "campaign_id", "campaign_version_id", "entry_index", "entry_id", "recipient_email", "from", "to", "cc", "bcc", "reply_to", "subject", "message_id_header", "build_status", "validation_status", "queue_status", "send_status", "delivery_channel_policy", "postbox_status", "imap_status", "attempt_count", "postbox_attempt_count", "postbox_target_count", "postbox_targets", "queued_at", "outcome_unknown_at", "sent_at", "last_error", "eml_size_bytes", "eml_sha256", "issues_count", "attachment_config_count", "matched_file_count", "attachment_names", "latest_smtp_attempt_number", "latest_smtp_status", "latest_smtp_status_code", "latest_smtp_started_at", "latest_smtp_finished_at", "latest_imap_attempt_number", "latest_imap_status", "latest_imap_folder", "latest_imap_created_at", "latest_imap_updated_at", ] if include_diagnostics: sent_at_index = fieldnames.index("outcome_unknown_at") fieldnames[sent_at_index:sent_at_index] = ["claimed_at", "smtp_started_at"] buffer = io.StringIO() writer = csv.DictWriter(buffer, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) return buffer.getvalue() def _latest_attempts_for_jobs( session: Session, model: type[Any], job_ids: list[str], *, batch_size: int = 500, ) -> dict[str, Any]: latest: dict[str, Any] = {} for offset in range(0, len(job_ids), batch_size): batch = job_ids[offset : offset + batch_size] attempts = ( session.query(model) .filter(model.job_id.in_(batch)) .order_by(model.job_id.asc(), model.attempt_number.asc()) .yield_per(batch_size) ) for attempt in attempts: current = latest.get(attempt.job_id) if ( current is None or (attempt.attempt_number or 0) >= (current.attempt_number or 0) ): latest[attempt.job_id] = attempt return latest