1051 lines
37 KiB
Python
1051 lines
37 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
import logging
|
|
import math
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.settings import settings as core_settings
|
|
|
|
from govoplan_campaign.backend.db.models import (
|
|
Campaign,
|
|
CampaignIssue,
|
|
CampaignJob,
|
|
CampaignVersion,
|
|
ImapAppendAttempt,
|
|
PostboxDeliveryAttempt,
|
|
SendAttempt,
|
|
)
|
|
from govoplan_campaign.backend.sending.execution import ExecutionSnapshot
|
|
from govoplan_campaign.backend.response_security import (
|
|
public_campaign_payload,
|
|
public_delivery_result_message,
|
|
public_source_filename,
|
|
)
|
|
|
|
|
|
class CampaignReportError(RuntimeError):
|
|
pass
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
CAMPAIGN_JSON_JOB_LIMIT = 5_000
|
|
CAMPAIGN_CSV_JOB_LIMIT = 100_000
|
|
|
|
|
|
def _utcnow_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def _counter(values: list[str | None]) -> dict[str, int]:
|
|
return dict(Counter(value or "unknown" for value in values))
|
|
|
|
|
|
def _get_campaign(session: Session, *, tenant_id: str, campaign_id: str) -> Campaign:
|
|
campaign = session.query(Campaign).filter(Campaign.tenant_id == tenant_id, Campaign.id == campaign_id).one_or_none()
|
|
if not campaign:
|
|
raise CampaignReportError(f"Campaign not found or not accessible: {campaign_id}")
|
|
return campaign
|
|
|
|
|
|
def _selected_version(
|
|
session: Session,
|
|
campaign: Campaign,
|
|
version_id: str | None = None,
|
|
) -> CampaignVersion | None:
|
|
wanted = version_id or campaign.current_version_id
|
|
if not wanted:
|
|
return None
|
|
version = session.get(CampaignVersion, wanted)
|
|
if not version or version.campaign_id != campaign.id:
|
|
raise CampaignReportError(f"Campaign version not found or not part of campaign: {wanted}")
|
|
return version
|
|
|
|
|
|
def _version_info(
|
|
version: CampaignVersion | None,
|
|
*,
|
|
include_diagnostics: bool = False,
|
|
) -> dict[str, Any] | None:
|
|
if not version:
|
|
return None
|
|
return {
|
|
"id": version.id,
|
|
"version_number": version.version_number,
|
|
"schema_version": version.schema_version,
|
|
"source_filename": public_source_filename(version.source_filename),
|
|
"created_at": version.created_at.isoformat() if version.created_at else None,
|
|
"validation_summary": public_campaign_payload(
|
|
version.validation_summary,
|
|
include_diagnostics=include_diagnostics,
|
|
),
|
|
"build_summary": public_campaign_payload(
|
|
version.build_summary,
|
|
include_diagnostics=include_diagnostics,
|
|
),
|
|
"execution_snapshot_hash": version.execution_snapshot_hash,
|
|
"execution_snapshot_at": version.execution_snapshot_at.isoformat() if version.execution_snapshot_at else None,
|
|
"delivery_mode": version.delivery_mode,
|
|
"delivery_mode_selected_at": (
|
|
version.delivery_mode_selected_at.isoformat() if version.delivery_mode_selected_at else None
|
|
),
|
|
}
|
|
|
|
|
|
def _load_delivery_info(
|
|
version: CampaignVersion | None,
|
|
jobs: list[CampaignJob] | None = None,
|
|
*,
|
|
pending_job_count: int | None = None,
|
|
include_diagnostics: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Read deterministic delivery settings from the immutable execution snapshot."""
|
|
|
|
default = {
|
|
"rate_limit": {"messages_per_minute": None, "concurrency": None},
|
|
"imap_append_sent": {"enabled": None, "folder": None},
|
|
"retry": {"max_attempts": None, "backoff_seconds": []},
|
|
"execution_snapshot_hash": None,
|
|
"execution_snapshot_at": None,
|
|
"snapshot_version": None,
|
|
"built_at": None,
|
|
"job_count": 0,
|
|
"queueable_job_count": 0,
|
|
"estimated_remaining_send_seconds": None,
|
|
"estimated_remaining_send_human": None,
|
|
"delivery_mode": getattr(version, "delivery_mode", None) if version else None,
|
|
"delivery_mode_selected_at": (
|
|
getattr(version, "delivery_mode_selected_at", None).isoformat()
|
|
if version and getattr(version, "delivery_mode_selected_at", None)
|
|
else None
|
|
),
|
|
}
|
|
if include_diagnostics:
|
|
default.update(
|
|
{
|
|
"build_token": None, # nosec B105 - absent report value, not a credential.
|
|
"job_manifest_sha256": None,
|
|
"effective_policy_sha256": None,
|
|
"smtp_transport_revision": None,
|
|
"imap_transport_revision": None,
|
|
"background_workers_enabled": bool(core_settings.celery_enabled),
|
|
}
|
|
)
|
|
if not version or not isinstance(version.execution_snapshot, dict):
|
|
default["load_error"] = "No execution snapshot exists; rebuild the current locked version before delivery."
|
|
return default
|
|
try:
|
|
snapshot = ExecutionSnapshot.model_validate(version.execution_snapshot)
|
|
except Exception: # pragma: no cover - reporting should remain available
|
|
logger.warning("Campaign execution snapshot could not be loaded for a report")
|
|
default["load_error"] = "Execution snapshot is invalid; rebuild the selected version before delivery."
|
|
return default
|
|
|
|
messages_per_minute = snapshot.delivery.rate_limit.messages_per_minute
|
|
if pending_job_count is None:
|
|
pending_job_count = sum(
|
|
1
|
|
for job in jobs or []
|
|
if job.send_status in {"queued", "claimed", "sending"}
|
|
)
|
|
estimated_seconds = None
|
|
if messages_per_minute and pending_job_count:
|
|
estimated_seconds = int(
|
|
math.ceil((pending_job_count / messages_per_minute) * 60)
|
|
)
|
|
|
|
result = {
|
|
"rate_limit": {
|
|
"messages_per_minute": messages_per_minute,
|
|
"concurrency": snapshot.delivery.rate_limit.concurrency,
|
|
},
|
|
"imap_append_sent": {
|
|
"enabled": snapshot.delivery.imap_append_sent.enabled,
|
|
"folder": snapshot.delivery.imap_append_sent.folder,
|
|
},
|
|
"retry": {
|
|
"max_attempts": snapshot.delivery.retry.max_attempts,
|
|
"backoff_seconds": snapshot.delivery.retry.backoff_seconds,
|
|
},
|
|
"execution_snapshot_hash": version.execution_snapshot_hash,
|
|
"execution_snapshot_at": version.execution_snapshot_at.isoformat() if version.execution_snapshot_at else None,
|
|
"snapshot_version": snapshot.snapshot_version,
|
|
"built_at": snapshot.built_at,
|
|
"job_count": snapshot.job_count,
|
|
"queueable_job_count": snapshot.queueable_job_count,
|
|
"estimated_remaining_send_seconds": estimated_seconds,
|
|
"estimated_remaining_send_human": _human_duration(estimated_seconds),
|
|
"delivery_mode": version.delivery_mode,
|
|
"delivery_mode_selected_at": (
|
|
version.delivery_mode_selected_at.isoformat() if version.delivery_mode_selected_at else None
|
|
),
|
|
}
|
|
if include_diagnostics:
|
|
result.update(
|
|
{
|
|
"build_token": snapshot.build_token,
|
|
"job_manifest_sha256": snapshot.job_manifest_sha256,
|
|
"effective_policy_sha256": snapshot.effective_policy_sha256,
|
|
"smtp_transport_revision": snapshot.smtp_transport_revision,
|
|
"imap_transport_revision": snapshot.imap_transport_revision,
|
|
"background_workers_enabled": bool(core_settings.celery_enabled),
|
|
}
|
|
)
|
|
return result
|
|
|
|
|
|
def _human_duration(seconds: int | None) -> str | None:
|
|
if seconds is None:
|
|
return None
|
|
if seconds < 60:
|
|
return f"{seconds}s"
|
|
minutes, sec = divmod(seconds, 60)
|
|
if minutes < 60:
|
|
return f"{minutes}m {sec}s" if sec else f"{minutes}m"
|
|
hours, minute = divmod(minutes, 60)
|
|
return f"{hours}h {minute}m" if minute else f"{hours}h"
|
|
|
|
|
|
def _issue_summary_from_jobs(jobs: list[CampaignJob]) -> dict[str, Any]:
|
|
severity_counter: Counter[str] = Counter()
|
|
code_counter: Counter[str] = Counter()
|
|
behavior_counter: Counter[str] = Counter()
|
|
total = 0
|
|
for job in jobs:
|
|
for issue in job.issues_snapshot or []:
|
|
if not isinstance(issue, dict):
|
|
continue
|
|
total += 1
|
|
severity_counter[issue.get("severity") or "unknown"] += 1
|
|
code_counter[issue.get("code") or "unknown"] += 1
|
|
if issue.get("behavior"):
|
|
behavior_counter[issue["behavior"]] += 1
|
|
return {
|
|
"total": total,
|
|
"by_severity": dict(severity_counter),
|
|
"by_code": dict(code_counter),
|
|
"by_behavior": dict(behavior_counter),
|
|
}
|
|
|
|
|
|
def _attachment_summary(jobs: list[CampaignJob]) -> dict[str, Any]:
|
|
status_counter: Counter[str] = Counter()
|
|
behavior_counter: Counter[str] = Counter()
|
|
total_configs = 0
|
|
total_matched_files = 0
|
|
zip_enabled = 0
|
|
missing = 0
|
|
ambiguous = 0
|
|
for job in jobs:
|
|
for attachment in job.resolved_attachments or []:
|
|
if not isinstance(attachment, dict):
|
|
continue
|
|
total_configs += 1
|
|
status = attachment.get("status") or "unknown"
|
|
status_counter[status] += 1
|
|
if attachment.get("behavior"):
|
|
behavior_counter[attachment["behavior"]] += 1
|
|
matches = attachment.get("matches") or []
|
|
if isinstance(matches, list):
|
|
total_matched_files += len(matches)
|
|
if attachment.get("zip_enabled"):
|
|
zip_enabled += 1
|
|
if status == "missing":
|
|
missing += 1
|
|
if status == "ambiguous":
|
|
ambiguous += 1
|
|
return {
|
|
"total_attachment_configs": total_configs,
|
|
"total_matched_files": total_matched_files,
|
|
"zip_enabled_configs": zip_enabled,
|
|
"missing_configs": missing,
|
|
"ambiguous_configs": ambiguous,
|
|
"by_status": dict(status_counter),
|
|
"by_behavior": dict(behavior_counter),
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _JobReportAggregate:
|
|
total: int = 0
|
|
pending: int = 0
|
|
queueable: int = 0
|
|
queueable_unattempted: int = 0
|
|
retryable: int = 0
|
|
cancellable: int = 0
|
|
needs_attention: int = 0
|
|
build: Counter[str] = field(default_factory=Counter)
|
|
validation: Counter[str] = field(default_factory=Counter)
|
|
queue: Counter[str] = field(default_factory=Counter)
|
|
send: Counter[str] = field(default_factory=Counter)
|
|
postbox: Counter[str] = field(default_factory=Counter)
|
|
imap: Counter[str] = field(default_factory=Counter)
|
|
issue_total: int = 0
|
|
issue_severity: Counter[str] = field(default_factory=Counter)
|
|
issue_code: Counter[str] = field(default_factory=Counter)
|
|
issue_behavior: Counter[str] = field(default_factory=Counter)
|
|
attachment_total: int = 0
|
|
attachment_matches: int = 0
|
|
attachment_zip_enabled: int = 0
|
|
attachment_missing: int = 0
|
|
attachment_ambiguous: int = 0
|
|
attachment_status: Counter[str] = field(default_factory=Counter)
|
|
attachment_behavior: Counter[str] = field(default_factory=Counter)
|
|
recent_failures: list[CampaignJob] = field(default_factory=list)
|
|
|
|
def add(
|
|
self,
|
|
job: CampaignJob,
|
|
*,
|
|
retry_max_attempts: int | None,
|
|
include_recent_failures: bool,
|
|
) -> None:
|
|
self.total += 1
|
|
self.build[job.build_status or "unknown"] += 1
|
|
self.validation[job.validation_status or "unknown"] += 1
|
|
self.queue[job.queue_status or "unknown"] += 1
|
|
self.send[job.send_status or "unknown"] += 1
|
|
self.postbox[job.postbox_status or "unknown"] += 1
|
|
self.imap[job.imap_status or "unknown"] += 1
|
|
if job.send_status in {"queued", "claimed", "sending"}:
|
|
self.pending += 1
|
|
if (
|
|
job.validation_status in {"ready", "warning"}
|
|
and job.build_status == "built"
|
|
):
|
|
self.queueable += 1
|
|
if (
|
|
job.attempt_count == 0
|
|
and job.postbox_attempt_count == 0
|
|
and job.send_status in {"not_queued", "cancelled"}
|
|
and job.validation_status in {"ready", "warning"}
|
|
and job.build_status == "built"
|
|
):
|
|
self.queueable_unattempted += 1
|
|
if (
|
|
job.send_status in {"failed_temporary", "partially_accepted"}
|
|
and (
|
|
retry_max_attempts is None
|
|
or job.attempt_count < retry_max_attempts
|
|
)
|
|
):
|
|
self.retryable += 1
|
|
if job.send_status not in {
|
|
"skipped",
|
|
"smtp_accepted",
|
|
"postbox_accepted",
|
|
"delivered",
|
|
"partially_accepted",
|
|
"sent",
|
|
"outcome_unknown",
|
|
"claimed",
|
|
"sending",
|
|
"cancelled",
|
|
}:
|
|
self.cancellable += 1
|
|
if _job_needs_attention(job):
|
|
self.needs_attention += 1
|
|
self._add_issues(job)
|
|
self._add_attachments(job)
|
|
if include_recent_failures and _job_is_recent_failure(job):
|
|
self.recent_failures.append(job)
|
|
self.recent_failures.sort(
|
|
key=lambda item: item.updated_at or item.created_at,
|
|
reverse=True,
|
|
)
|
|
del self.recent_failures[20:]
|
|
|
|
def _add_issues(self, job: CampaignJob) -> None:
|
|
for issue in job.issues_snapshot or []:
|
|
if not isinstance(issue, dict):
|
|
continue
|
|
self.issue_total += 1
|
|
self.issue_severity[issue.get("severity") or "unknown"] += 1
|
|
self.issue_code[issue.get("code") or "unknown"] += 1
|
|
if issue.get("behavior"):
|
|
self.issue_behavior[str(issue["behavior"])] += 1
|
|
|
|
def _add_attachments(self, job: CampaignJob) -> None:
|
|
for attachment in job.resolved_attachments or []:
|
|
if not isinstance(attachment, dict):
|
|
continue
|
|
self.attachment_total += 1
|
|
status_value = str(attachment.get("status") or "unknown")
|
|
self.attachment_status[status_value] += 1
|
|
if attachment.get("behavior"):
|
|
self.attachment_behavior[str(attachment["behavior"])] += 1
|
|
matches = attachment.get("matches")
|
|
if isinstance(matches, list):
|
|
self.attachment_matches += len(matches)
|
|
if attachment.get("zip_enabled"):
|
|
self.attachment_zip_enabled += 1
|
|
if status_value == "missing":
|
|
self.attachment_missing += 1
|
|
if status_value == "ambiguous":
|
|
self.attachment_ambiguous += 1
|
|
|
|
|
|
def _job_needs_attention(job: CampaignJob) -> bool:
|
|
return (
|
|
job.validation_status in {"needs_review", "blocked"}
|
|
or job.send_status
|
|
in {
|
|
"failed_temporary",
|
|
"failed_permanent",
|
|
"partially_accepted",
|
|
"outcome_unknown",
|
|
"claimed",
|
|
"sending",
|
|
}
|
|
or job.postbox_status
|
|
in {
|
|
"partially_accepted",
|
|
"rejected_temporary",
|
|
"rejected_permanent",
|
|
"outcome_unknown",
|
|
}
|
|
or job.imap_status == "failed"
|
|
)
|
|
|
|
|
|
def _job_is_recent_failure(job: CampaignJob) -> bool:
|
|
return bool(
|
|
job.last_error
|
|
or str(job.send_status).startswith("failed")
|
|
or job.send_status == "partially_accepted"
|
|
or job.postbox_status
|
|
in {
|
|
"partially_accepted",
|
|
"rejected_temporary",
|
|
"rejected_permanent",
|
|
"outcome_unknown",
|
|
}
|
|
or job.imap_status == "failed"
|
|
)
|
|
|
|
|
|
def _recent_failures(jobs: list[CampaignJob], *, limit: int = 20) -> list[dict[str, Any]]:
|
|
failed = [job for job in jobs if _job_is_recent_failure(job)]
|
|
failed.sort(key=lambda job: job.updated_at or job.created_at, reverse=True)
|
|
return [
|
|
{
|
|
"job_id": job.id,
|
|
"entry_index": job.entry_index,
|
|
"entry_id": job.entry_id,
|
|
"recipient_email": job.recipient_email,
|
|
"validation_status": job.validation_status,
|
|
"send_status": job.send_status,
|
|
"postbox_status": job.postbox_status,
|
|
"imap_status": job.imap_status,
|
|
"attempt_count": job.attempt_count,
|
|
"postbox_attempt_count": job.postbox_attempt_count,
|
|
"last_error": public_delivery_result_message(
|
|
last_error=job.last_error,
|
|
send_status=job.send_status,
|
|
imap_status=job.imap_status,
|
|
postbox_status=job.postbox_status,
|
|
),
|
|
"updated_at": job.updated_at.isoformat() if job.updated_at else None,
|
|
}
|
|
for job in failed[:limit]
|
|
]
|
|
|
|
|
|
def _job_row(job: CampaignJob, *, include_diagnostics: bool = False) -> dict[str, Any]:
|
|
row = {
|
|
"job_id": job.id,
|
|
"entry_index": job.entry_index,
|
|
"entry_id": job.entry_id,
|
|
"recipient_email": job.recipient_email,
|
|
"subject": job.subject,
|
|
"build_status": job.build_status,
|
|
"validation_status": job.validation_status,
|
|
"queue_status": job.queue_status,
|
|
"send_status": job.send_status,
|
|
"delivery_channel_policy": getattr(
|
|
job,
|
|
"delivery_channel_policy",
|
|
"mail",
|
|
),
|
|
"postbox_status": getattr(
|
|
job,
|
|
"postbox_status",
|
|
"not_requested",
|
|
),
|
|
"imap_status": job.imap_status,
|
|
"attempt_count": job.attempt_count,
|
|
"postbox_attempt_count": getattr(job, "postbox_attempt_count", 0),
|
|
"postbox_target_count": len(
|
|
getattr(job, "resolved_postbox_targets", None) or []
|
|
),
|
|
"queued_at": job.queued_at.isoformat() if job.queued_at else None,
|
|
"outcome_unknown_at": job.outcome_unknown_at.isoformat() if job.outcome_unknown_at else None,
|
|
"sent_at": job.sent_at.isoformat() if job.sent_at else None,
|
|
"last_error": public_delivery_result_message(
|
|
last_error=job.last_error,
|
|
send_status=job.send_status,
|
|
imap_status=job.imap_status,
|
|
postbox_status=getattr(job, "postbox_status", "not_requested"),
|
|
),
|
|
"eml_size_bytes": job.eml_size_bytes,
|
|
"eml_sha256": job.eml_sha256,
|
|
"issues_count": len(job.issues_snapshot or []),
|
|
"attachment_config_count": len(job.resolved_attachments or []),
|
|
"matched_file_count": sum(len(item.get("matches") or []) for item in (job.resolved_attachments or []) if isinstance(item, dict)),
|
|
}
|
|
if include_diagnostics:
|
|
row.update(
|
|
{
|
|
"claimed_at": job.claimed_at.isoformat() if job.claimed_at else None,
|
|
"smtp_started_at": job.smtp_started_at.isoformat() if job.smtp_started_at else None,
|
|
}
|
|
)
|
|
return row
|
|
|
|
|
|
def _address_summary(value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, str):
|
|
return value
|
|
if isinstance(value, dict):
|
|
email = str(value.get("email") or "").strip()
|
|
name = str(value.get("name") or "").strip()
|
|
if name and email:
|
|
return f"{name} <{email}>"
|
|
return email or name
|
|
if isinstance(value, list):
|
|
return "; ".join(item for item in (_address_summary(item) for item in value) if item)
|
|
return str(value)
|
|
|
|
|
|
def _latest_by_job_id(attempts: list[Any]) -> dict[str, Any]:
|
|
latest: dict[str, Any] = {}
|
|
for attempt in attempts:
|
|
current = latest.get(attempt.job_id)
|
|
if current is None or (attempt.attempt_number or 0) >= (current.attempt_number or 0):
|
|
latest[attempt.job_id] = attempt
|
|
return latest
|
|
|
|
|
|
def _attachment_names(attachments: list[dict[str, Any]] | None) -> str:
|
|
names: list[str] = []
|
|
for attachment in attachments or []:
|
|
if not isinstance(attachment, dict):
|
|
continue
|
|
for key in ("message_filenames", "zip_entry_names"):
|
|
values = attachment.get(key)
|
|
if isinstance(values, list):
|
|
names.extend(str(value) for value in values if value)
|
|
zip_filename = attachment.get("zip_filename")
|
|
if zip_filename:
|
|
names.append(str(zip_filename))
|
|
matches = attachment.get("matches")
|
|
if isinstance(matches, list):
|
|
for match in matches:
|
|
if isinstance(match, dict):
|
|
filename = match.get("filename") or match.get("name") or match.get("display_name")
|
|
if filename:
|
|
names.append(str(filename))
|
|
elif match:
|
|
names.append(str(match).rsplit("/", 1)[-1])
|
|
seen: set[str] = set()
|
|
deduplicated = []
|
|
for name in names:
|
|
if name and name not in seen:
|
|
seen.add(name)
|
|
deduplicated.append(name)
|
|
return "; ".join(deduplicated)
|
|
|
|
|
|
def _job_evidence_row(
|
|
job: CampaignJob,
|
|
*,
|
|
latest_smtp: SendAttempt | None = None,
|
|
latest_imap: ImapAppendAttempt | None = None,
|
|
include_diagnostics: bool = False,
|
|
) -> dict[str, Any]:
|
|
row = _job_row(job, include_diagnostics=include_diagnostics)
|
|
recipients = job.resolved_recipients or {}
|
|
row.update({
|
|
"campaign_id": job.campaign_id,
|
|
"campaign_version_id": job.campaign_version_id,
|
|
"message_id_header": job.message_id_header,
|
|
"from": _address_summary(recipients.get("from")),
|
|
"to": _address_summary(recipients.get("to")),
|
|
"cc": _address_summary(recipients.get("cc")),
|
|
"bcc": _address_summary(recipients.get("bcc")),
|
|
"reply_to": _address_summary(recipients.get("reply_to")),
|
|
"postbox_targets": "; ".join(
|
|
str(target.get("address") or target.get("name") or target.get("postbox_id") or "")
|
|
for target in (
|
|
getattr(job, "resolved_postbox_targets", None) or []
|
|
)
|
|
if isinstance(target, dict)
|
|
),
|
|
"attachment_names": _attachment_names(job.resolved_attachments),
|
|
"latest_smtp_attempt_number": latest_smtp.attempt_number if latest_smtp else None,
|
|
"latest_smtp_status": latest_smtp.status if latest_smtp else None,
|
|
"latest_smtp_status_code": latest_smtp.smtp_status_code if latest_smtp else None,
|
|
"latest_smtp_started_at": latest_smtp.started_at.isoformat() if latest_smtp and latest_smtp.started_at else None,
|
|
"latest_smtp_finished_at": latest_smtp.finished_at.isoformat() if latest_smtp and latest_smtp.finished_at else None,
|
|
"latest_imap_attempt_number": latest_imap.attempt_number if latest_imap else None,
|
|
"latest_imap_status": latest_imap.status if latest_imap else None,
|
|
"latest_imap_folder": latest_imap.folder if latest_imap else None,
|
|
"latest_imap_created_at": latest_imap.created_at.isoformat() if latest_imap and latest_imap.created_at else None,
|
|
"latest_imap_updated_at": latest_imap.updated_at.isoformat() if latest_imap and latest_imap.updated_at else None,
|
|
})
|
|
return row
|
|
|
|
|
|
def generate_campaign_report(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version_id: str | None = None,
|
|
include_jobs: bool = False,
|
|
include_recent_failures: bool = False,
|
|
include_diagnostics: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Generate a dashboard/report payload for one campaign.
|
|
|
|
The shape is intentionally web-UI friendly: status counters for cards,
|
|
issue/attachment summaries for review panels, and optional job rows for
|
|
tables/export.
|
|
"""
|
|
|
|
campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
|
|
version = _selected_version(session, campaign, version_id)
|
|
jobs = _report_jobs(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
campaign_id=campaign.id,
|
|
version=version,
|
|
)
|
|
aggregate = _JobReportAggregate()
|
|
job_rows: list[dict[str, Any]] = []
|
|
retry_max_attempts = _retry_max_attempts(version)
|
|
iterator = jobs.yield_per(500) if hasattr(jobs, "yield_per") else jobs
|
|
for job in iterator:
|
|
aggregate.add(
|
|
job,
|
|
retry_max_attempts=retry_max_attempts,
|
|
include_recent_failures=include_recent_failures,
|
|
)
|
|
if include_jobs:
|
|
if len(job_rows) >= CAMPAIGN_JSON_JOB_LIMIT:
|
|
raise CampaignReportError(
|
|
"The recipient-level JSON report exceeds the safe row "
|
|
f"limit of {CAMPAIGN_JSON_JOB_LIMIT}; use the CSV export "
|
|
"or the paginated recipient table."
|
|
)
|
|
job_rows.append(
|
|
_job_row(job, include_diagnostics=include_diagnostics)
|
|
)
|
|
report = _campaign_report_payload(
|
|
session,
|
|
campaign=campaign,
|
|
version=version,
|
|
aggregate=aggregate,
|
|
tenant_id=tenant_id,
|
|
include_recent_failures=include_recent_failures,
|
|
include_diagnostics=include_diagnostics,
|
|
)
|
|
if include_jobs:
|
|
report["jobs"] = job_rows
|
|
return report
|
|
|
|
|
|
def _report_jobs(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version: CampaignVersion | None,
|
|
) -> Any:
|
|
jobs_query = session.query(CampaignJob).filter(
|
|
CampaignJob.tenant_id == tenant_id,
|
|
CampaignJob.campaign_id == campaign_id,
|
|
)
|
|
if version:
|
|
jobs_query = jobs_query.filter(CampaignJob.campaign_version_id == version.id)
|
|
else:
|
|
jobs_query = jobs_query.filter(False)
|
|
return jobs_query.order_by(CampaignJob.entry_index.asc())
|
|
|
|
|
|
def _campaign_report_payload(
|
|
session: Session,
|
|
*,
|
|
campaign: Campaign,
|
|
version: CampaignVersion | None,
|
|
aggregate: _JobReportAggregate,
|
|
tenant_id: str,
|
|
include_recent_failures: bool,
|
|
include_diagnostics: bool,
|
|
) -> dict[str, Any]:
|
|
report = {
|
|
"generated_at": _utcnow_iso(),
|
|
"campaign": _campaign_report_campaign_payload(campaign),
|
|
"current_version": _version_info(version, include_diagnostics=include_diagnostics),
|
|
"selected_version_id": version.id if version else None,
|
|
"cards": _campaign_report_cards_from_aggregate(version, aggregate),
|
|
"status_counts": _campaign_report_status_counts_from_aggregate(
|
|
version,
|
|
aggregate,
|
|
),
|
|
"issues": {
|
|
"total": aggregate.issue_total,
|
|
"by_severity": dict(aggregate.issue_severity),
|
|
"by_code": dict(aggregate.issue_code),
|
|
"by_behavior": dict(aggregate.issue_behavior),
|
|
"persisted_campaign_issue_count": _persisted_campaign_issue_count(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
campaign_id=campaign.id,
|
|
version=version,
|
|
),
|
|
},
|
|
"attachments": {
|
|
"total_attachment_configs": aggregate.attachment_total,
|
|
"total_matched_files": aggregate.attachment_matches,
|
|
"zip_enabled_configs": aggregate.attachment_zip_enabled,
|
|
"missing_configs": aggregate.attachment_missing,
|
|
"ambiguous_configs": aggregate.attachment_ambiguous,
|
|
"by_status": dict(aggregate.attachment_status),
|
|
"by_behavior": dict(aggregate.attachment_behavior),
|
|
},
|
|
"attempts": _campaign_report_attempt_counts(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
campaign_id=campaign.id,
|
|
version=version,
|
|
),
|
|
"delivery": _load_delivery_info(
|
|
version,
|
|
pending_job_count=aggregate.pending,
|
|
include_diagnostics=include_diagnostics,
|
|
),
|
|
}
|
|
if include_recent_failures:
|
|
report["recent_failures"] = _recent_failures(
|
|
aggregate.recent_failures,
|
|
)
|
|
return report
|
|
|
|
|
|
def _campaign_report_campaign_payload(campaign: Campaign) -> dict[str, Any]:
|
|
return {
|
|
"id": campaign.id,
|
|
"external_id": campaign.external_id,
|
|
"name": campaign.name,
|
|
"description": campaign.description,
|
|
"status": campaign.status,
|
|
"created_at": campaign.created_at.isoformat() if campaign.created_at else None,
|
|
"updated_at": campaign.updated_at.isoformat() if campaign.updated_at else None,
|
|
}
|
|
|
|
|
|
def _campaign_report_attempt_counts(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version: CampaignVersion | None,
|
|
) -> dict[str, int]:
|
|
def count(model: type[Any]) -> int:
|
|
query = (
|
|
session.query(model)
|
|
.join(CampaignJob, model.job_id == CampaignJob.id)
|
|
.filter(
|
|
CampaignJob.tenant_id == tenant_id,
|
|
CampaignJob.campaign_id == campaign_id,
|
|
)
|
|
)
|
|
if version is not None:
|
|
query = query.filter(
|
|
CampaignJob.campaign_version_id == version.id,
|
|
)
|
|
else:
|
|
query = query.filter(False)
|
|
return int(query.count())
|
|
|
|
return {
|
|
"send_attempts": count(SendAttempt),
|
|
"imap_append_attempts": count(ImapAppendAttempt),
|
|
"postbox_delivery_attempts": count(PostboxDeliveryAttempt),
|
|
}
|
|
|
|
|
|
def _persisted_campaign_issue_count(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version: CampaignVersion | None,
|
|
) -> int:
|
|
issue_query = session.query(CampaignIssue).filter(
|
|
CampaignIssue.tenant_id == tenant_id,
|
|
CampaignIssue.campaign_id == campaign_id,
|
|
)
|
|
if version:
|
|
issue_query = issue_query.filter(CampaignIssue.campaign_version_id == version.id)
|
|
else:
|
|
issue_query = issue_query.filter(False)
|
|
return int(issue_query.count())
|
|
|
|
|
|
def _campaign_report_status_counts(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, dict[str, int]]:
|
|
aggregate = _aggregate_job_list(version, jobs)
|
|
return _campaign_report_status_counts_from_aggregate(version, aggregate)
|
|
|
|
|
|
def _campaign_report_status_counts_from_aggregate(
|
|
version: CampaignVersion | None,
|
|
aggregate: _JobReportAggregate,
|
|
) -> dict[str, dict[str, int]]:
|
|
validation_counts = dict(aggregate.validation)
|
|
inactive_entries = _inactive_entry_count(version)
|
|
if inactive_entries:
|
|
validation_counts["inactive"] = inactive_entries
|
|
return {
|
|
"build": dict(aggregate.build),
|
|
"validation": validation_counts,
|
|
"queue": dict(aggregate.queue),
|
|
"send": dict(aggregate.send),
|
|
"postbox": dict(aggregate.postbox),
|
|
"imap": dict(aggregate.imap),
|
|
}
|
|
|
|
|
|
def _campaign_report_cards_from_aggregate(
|
|
version: CampaignVersion | None,
|
|
aggregate: _JobReportAggregate,
|
|
) -> dict[str, object]:
|
|
send_counts = aggregate.send
|
|
imap_counts = aggregate.imap
|
|
sent = (
|
|
send_counts.get("sent", 0)
|
|
+ send_counts.get("smtp_accepted", 0)
|
|
+ send_counts.get("postbox_accepted", 0)
|
|
+ send_counts.get("delivered", 0)
|
|
+ send_counts.get("partially_accepted", 0)
|
|
)
|
|
failed = (
|
|
send_counts.get("failed_temporary", 0)
|
|
+ send_counts.get("failed_permanent", 0)
|
|
)
|
|
outcome_unknown = send_counts.get("outcome_unknown", 0)
|
|
not_attempted = send_counts.get("not_queued", 0)
|
|
cancelled = send_counts.get("cancelled", 0)
|
|
inactive_entries = _inactive_entry_count(version)
|
|
return {
|
|
"jobs_total": aggregate.total,
|
|
"inactive": inactive_entries,
|
|
"queueable": aggregate.queueable,
|
|
"queueable_unattempted": aggregate.queueable_unattempted,
|
|
"retryable": aggregate.retryable,
|
|
"cancellable": aggregate.cancellable,
|
|
"needs_attention": aggregate.needs_attention,
|
|
"sent": sent,
|
|
"smtp_accepted": (
|
|
send_counts.get("sent", 0)
|
|
+ send_counts.get("smtp_accepted", 0)
|
|
),
|
|
"postbox_accepted": send_counts.get("postbox_accepted", 0),
|
|
"delivered": send_counts.get("delivered", 0),
|
|
"partially_accepted": send_counts.get("partially_accepted", 0),
|
|
"failed": failed,
|
|
"outcome_unknown": outcome_unknown,
|
|
"not_attempted": not_attempted,
|
|
"skipped": send_counts.get("skipped", 0),
|
|
"queued_or_active": aggregate.pending,
|
|
"cancelled": cancelled,
|
|
"partially_completed": bool(
|
|
send_counts.get("partially_accepted", 0)
|
|
or sent
|
|
and (failed or outcome_unknown or not_attempted or cancelled)
|
|
),
|
|
"imap_appended": imap_counts.get("appended", 0),
|
|
"imap_failed": imap_counts.get("failed", 0),
|
|
"imap_skipped": imap_counts.get("skipped", 0),
|
|
}
|
|
|
|
|
|
def _campaign_report_cards(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, object]:
|
|
return _campaign_report_cards_from_aggregate(
|
|
version,
|
|
_aggregate_job_list(version, jobs),
|
|
)
|
|
|
|
|
|
def _aggregate_job_list(
|
|
version: CampaignVersion | None,
|
|
jobs: list[CampaignJob],
|
|
) -> _JobReportAggregate:
|
|
aggregate = _JobReportAggregate()
|
|
retry_max_attempts = _retry_max_attempts(version)
|
|
for job in jobs:
|
|
aggregate.add(
|
|
job,
|
|
retry_max_attempts=retry_max_attempts,
|
|
include_recent_failures=False,
|
|
)
|
|
return aggregate
|
|
|
|
|
|
def _retry_max_attempts(version: CampaignVersion | None) -> int | None:
|
|
if version is None or not isinstance(
|
|
getattr(version, "execution_snapshot", None),
|
|
dict,
|
|
):
|
|
return None
|
|
try:
|
|
return ExecutionSnapshot.model_validate(version.execution_snapshot).delivery.retry.max_attempts
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _inactive_entry_count(version: CampaignVersion | None) -> int:
|
|
build_summary = version.build_summary if version and isinstance(version.build_summary, dict) else {}
|
|
return int(build_summary.get("inactive_count") or build_summary.get("inactive_entries_count") or 0)
|
|
|
|
|
|
def generate_jobs_csv(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version_id: str | None = None,
|
|
include_diagnostics: bool = False,
|
|
) -> str:
|
|
campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
|
|
version = _selected_version(session, campaign, version_id)
|
|
jobs_query = session.query(CampaignJob).filter(
|
|
CampaignJob.tenant_id == tenant_id,
|
|
CampaignJob.campaign_id == campaign.id,
|
|
)
|
|
if version:
|
|
jobs_query = jobs_query.filter(CampaignJob.campaign_version_id == version.id)
|
|
else:
|
|
jobs_query = jobs_query.filter(False)
|
|
jobs = (
|
|
jobs_query
|
|
.order_by(CampaignJob.entry_index.asc())
|
|
.limit(CAMPAIGN_CSV_JOB_LIMIT + 1)
|
|
.all()
|
|
)
|
|
if len(jobs) > CAMPAIGN_CSV_JOB_LIMIT:
|
|
raise CampaignReportError(
|
|
"The campaign CSV export exceeds the safe row limit of "
|
|
f"{CAMPAIGN_CSV_JOB_LIMIT}. Split the export by recipient filter."
|
|
)
|
|
job_ids = [job.id for job in jobs]
|
|
latest_smtp = _latest_attempts_for_jobs(
|
|
session,
|
|
SendAttempt,
|
|
job_ids,
|
|
)
|
|
latest_imap = _latest_attempts_for_jobs(
|
|
session,
|
|
ImapAppendAttempt,
|
|
job_ids,
|
|
)
|
|
rows = [
|
|
_job_evidence_row(
|
|
job,
|
|
latest_smtp=latest_smtp.get(job.id),
|
|
latest_imap=latest_imap.get(job.id),
|
|
include_diagnostics=include_diagnostics,
|
|
)
|
|
for job in jobs
|
|
]
|
|
fieldnames = [
|
|
"job_id",
|
|
"campaign_id",
|
|
"campaign_version_id",
|
|
"entry_index",
|
|
"entry_id",
|
|
"recipient_email",
|
|
"from",
|
|
"to",
|
|
"cc",
|
|
"bcc",
|
|
"reply_to",
|
|
"subject",
|
|
"message_id_header",
|
|
"build_status",
|
|
"validation_status",
|
|
"queue_status",
|
|
"send_status",
|
|
"delivery_channel_policy",
|
|
"postbox_status",
|
|
"imap_status",
|
|
"attempt_count",
|
|
"postbox_attempt_count",
|
|
"postbox_target_count",
|
|
"postbox_targets",
|
|
"queued_at",
|
|
"outcome_unknown_at",
|
|
"sent_at",
|
|
"last_error",
|
|
"eml_size_bytes",
|
|
"eml_sha256",
|
|
"issues_count",
|
|
"attachment_config_count",
|
|
"matched_file_count",
|
|
"attachment_names",
|
|
"latest_smtp_attempt_number",
|
|
"latest_smtp_status",
|
|
"latest_smtp_status_code",
|
|
"latest_smtp_started_at",
|
|
"latest_smtp_finished_at",
|
|
"latest_imap_attempt_number",
|
|
"latest_imap_status",
|
|
"latest_imap_folder",
|
|
"latest_imap_created_at",
|
|
"latest_imap_updated_at",
|
|
]
|
|
if include_diagnostics:
|
|
sent_at_index = fieldnames.index("outcome_unknown_at")
|
|
fieldnames[sent_at_index:sent_at_index] = ["claimed_at", "smtp_started_at"]
|
|
buffer = io.StringIO()
|
|
writer = csv.DictWriter(buffer, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def _latest_attempts_for_jobs(
|
|
session: Session,
|
|
model: type[Any],
|
|
job_ids: list[str],
|
|
*,
|
|
batch_size: int = 500,
|
|
) -> dict[str, Any]:
|
|
latest: dict[str, Any] = {}
|
|
for offset in range(0, len(job_ids), batch_size):
|
|
batch = job_ids[offset : offset + batch_size]
|
|
attempts = (
|
|
session.query(model)
|
|
.filter(model.job_id.in_(batch))
|
|
.order_by(model.job_id.asc(), model.attempt_number.asc())
|
|
.yield_per(batch_size)
|
|
)
|
|
for attempt in attempts:
|
|
current = latest.get(attempt.job_id)
|
|
if (
|
|
current is None
|
|
or (attempt.attempt_number or 0)
|
|
>= (current.attempt_number or 0)
|
|
):
|
|
latest[attempt.job_id] = attempt
|
|
return latest
|