from __future__ import annotations import hashlib import json from dataclasses import asdict, dataclass from datetime import datetime, timezone from email import policy from email.parser import BytesParser from pathlib import Path from typing import Any from uuid import uuid4 from sqlalchemy.orm import Session from govoplan_core.settings import settings as core_settings from govoplan_campaign.backend.db.models import ( Campaign, CampaignJob, CampaignStatus, CampaignVersion, CampaignVersionWorkflowState, JobBuildStatus, JobImapStatus, JobQueueStatus, JobSendStatus, JobValidationStatus, ImapAppendAttempt, SendAttempt, ) from govoplan_campaign.backend.sending.execution import ExecutionSnapshot, ExecutionSnapshotError, ensure_execution_snapshot, runtime_imap_config, runtime_smtp_config from govoplan_campaign.backend.integrations import ( ImapAppendError, ImapConfigurationError, MailProfileError, SmtpConfigurationError, SmtpSendError, files_integration, mail_integration, ) class QueueingError(RuntimeError): pass class SendJobError(RuntimeError): pass @dataclass(frozen=True, slots=True) class QueueCampaignResult: campaign_id: str version_id: str queued_count: int skipped_count: int blocked_count: int enqueued_count: int dry_run: bool = False def as_dict(self) -> dict[str, Any]: return { "campaign_id": self.campaign_id, "version_id": self.version_id, "queued_count": self.queued_count, "skipped_count": self.skipped_count, "blocked_count": self.blocked_count, "enqueued_count": self.enqueued_count, "dry_run": self.dry_run, } @dataclass(frozen=True, slots=True) class SendCampaignNowResult: campaign_id: str version_id: str attempted_count: int sent_count: int failed_count: int outcome_unknown_count: int skipped_count: int dry_run: bool = False results: list[dict[str, Any]] | None = None def as_dict(self) -> dict[str, Any]: return { "campaign_id": self.campaign_id, "version_id": self.version_id, "attempted_count": self.attempted_count, "sent_count": self.sent_count, "failed_count": self.failed_count, "outcome_unknown_count": self.outcome_unknown_count, "skipped_count": self.skipped_count, "dry_run": self.dry_run, "results": self.results or [], } @dataclass(frozen=True, slots=True) class SendJobResult: job_id: str status: str attempt_number: int dry_run: bool = False message: str | None = None def as_dict(self) -> dict[str, Any]: return { "job_id": self.job_id, "status": self.status, "attempt_number": self.attempt_number, "dry_run": self.dry_run, "message": self.message, } @dataclass(frozen=True, slots=True) class AppendSentResult: job_id: str status: str attempt_number: int dry_run: bool = False folder: str | None = None message: str | None = None def as_dict(self) -> dict[str, Any]: return { "job_id": self.job_id, "status": self.status, "attempt_number": self.attempt_number, "dry_run": self.dry_run, "folder": self.folder, "message": self.message, } QUEUEABLE_VALIDATION_STATUSES = { JobValidationStatus.READY.value, JobValidationStatus.WARNING.value, } SMTP_ACCEPTED_STATUSES = {JobSendStatus.SMTP_ACCEPTED.value, JobSendStatus.SENT.value} AUTOMATICALLY_SENDABLE_STATUSES = {JobSendStatus.QUEUED.value} EXPLICIT_RETRY_STATUSES = {JobSendStatus.FAILED_TEMPORARY.value, JobSendStatus.FAILED_PERMANENT.value} def _version_user_lock_state(version: CampaignVersion) -> str | None: state = getattr(version, "user_lock_state", None) if state in {"temporary", "permanent"}: return state return "permanent" if version.published_at else None def _version_is_user_locked(version: CampaignVersion) -> bool: return _version_user_lock_state(version) is not None def _version_is_validated_and_locked(version: CampaignVersion) -> bool: validation = version.validation_summary if isinstance(version.validation_summary, dict) else {} return bool(version.locked_at and validation.get("ok") is True and not _version_is_user_locked(version)) def _ensure_version_validated_and_locked(version: CampaignVersion) -> None: state = _version_user_lock_state(version) if state == "temporary": raise QueueingError("This version has a temporary user lock. Unlock it before queueing, dry-run or sending.") if state == "permanent": raise QueueingError("This version is permanently user-locked. Create an editable copy instead.") if not _version_is_validated_and_locked(version): raise QueueingError("Campaign version must be validated and locked before building, queueing, dry-run or sending.") def _utcnow() -> datetime: return datetime.now(timezone.utc) def _get_campaign_for_tenant(session: Session, *, campaign_id: str, tenant_id: str) -> Campaign: campaign = session.query(Campaign).filter(Campaign.id == campaign_id, Campaign.tenant_id == tenant_id).one_or_none() if not campaign: raise QueueingError(f"Campaign not found or not accessible: {campaign_id}") return campaign def _get_version_for_campaign( session: Session, campaign: Campaign, version_id: str | None = None, ) -> CampaignVersion: wanted = version_id or campaign.current_version_id if not wanted: raise QueueingError("Campaign has no selected version") version = session.get(CampaignVersion, wanted) if not version or version.campaign_id != campaign.id: raise QueueingError(f"Campaign version not found or not part of campaign: {wanted}") return version def _get_current_version(session: Session, campaign: Campaign, version_id: str | None = None) -> CampaignVersion: version = _get_version_for_campaign(session, campaign, version_id) if campaign.current_version_id != version.id: raise QueueingError( "Historical campaign versions cannot start a new execution. Use the explicit retry or reconciliation actions for an existing execution." ) return version def _celery_enabled() -> bool: return bool(core_settings.celery_enabled) def _should_enqueue_celery(enqueue_celery: bool) -> bool: return bool(enqueue_celery and _celery_enabled()) def _celery_enqueue_send_job(job_id: str) -> None: from govoplan_core.celery_app import celery celery.send_task("multimailer.send_email", args=[job_id], queue="send_email") def _celery_enqueue_append_sent_job(job_id: str) -> None: from govoplan_core.celery_app import celery celery.send_task("multimailer.append_sent", args=[job_id], queue="append_sent") def queue_campaign_jobs( session: Session, *, tenant_id: str, campaign_id: str, version_id: str | None = None, enqueue_celery: bool = True, include_warnings: bool = True, dry_run: bool = False, ) -> QueueCampaignResult: """Move queueable DB jobs to QUEUED and optionally enqueue Celery tasks.""" campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) version = _get_current_version(session, campaign, version_id=version_id) _ensure_version_validated_and_locked(version) try: ensure_execution_snapshot(session, version) except ExecutionSnapshotError as exc: raise QueueingError(str(exc)) from exc allowed_validation = {JobValidationStatus.READY.value} if include_warnings: allowed_validation.add(JobValidationStatus.WARNING.value) jobs = ( session.query(CampaignJob) .filter(CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_version_id == version.id) .order_by(CampaignJob.entry_index.asc()) .all() ) if not jobs: raise QueueingError("Campaign version has no jobs. Build messages before queueing.") queued: list[CampaignJob] = [] skipped_count = 0 blocked_count = 0 for job in jobs: if job.send_status in SMTP_ACCEPTED_STATUSES | { JobSendStatus.CLAIMED.value, JobSendStatus.SENDING.value, JobSendStatus.OUTCOME_UNKNOWN.value, JobSendStatus.FAILED_TEMPORARY.value, JobSendStatus.FAILED_PERMANENT.value, JobSendStatus.CANCELLED.value, }: # Initial queueing never doubles as retry/reconciliation. Those # states require the dedicated explicit actions below. skipped_count += 1 continue if job.queue_status in {JobQueueStatus.CANCELLED.value, JobQueueStatus.SENDING.value, JobQueueStatus.PAUSED.value}: skipped_count += 1 continue if job.build_status != JobBuildStatus.BUILT.value or job.validation_status not in allowed_validation: blocked_count += 1 continue if not job.eml_local_path and not job.eml_storage_key: job.last_error = "Job has no generated EML path/storage key. Rebuild with write_eml enabled before queueing." blocked_count += 1 continue queued.append(job) if not dry_run: job.queue_status = JobQueueStatus.QUEUED.value job.send_status = JobSendStatus.QUEUED.value job.queued_at = _utcnow() job.claimed_at = None job.claim_token = None job.smtp_started_at = None job.outcome_unknown_at = None job.last_error = None session.add(job) if not dry_run: if queued: campaign.status = CampaignStatus.QUEUED.value version.workflow_state = CampaignVersionWorkflowState.QUEUED.value if version.locked_at is None: version.locked_at = _utcnow() session.add(version) session.add(campaign) session.commit() enqueued_count = 0 if _should_enqueue_celery(enqueue_celery) and not dry_run: for job in queued: _celery_enqueue_send_job(job.id) enqueued_count += 1 return QueueCampaignResult( campaign_id=campaign.id, version_id=version.id, queued_count=len(queued), skipped_count=skipped_count, blocked_count=blocked_count, enqueued_count=enqueued_count, dry_run=dry_run, ) def send_campaign_now( session: Session, *, tenant_id: str, campaign_id: str, version_id: str | None = None, include_warnings: bool = True, dry_run: bool = False, use_rate_limit: bool = True, enqueue_imap_task: bool = False, ) -> SendCampaignNowResult: """Queue and send a small campaign synchronously. This is intended for WebUI test/small-campaign flows. Large campaigns can still use queue_campaign_jobs with Celery workers. """ campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) version = _get_current_version(session, campaign, version_id=version_id) queue_result = queue_campaign_jobs( session, tenant_id=tenant_id, campaign_id=campaign.id, version_id=version.id, include_warnings=include_warnings, enqueue_celery=False, dry_run=dry_run, ) if dry_run: return SendCampaignNowResult( campaign_id=campaign.id, version_id=version.id, attempted_count=0, sent_count=0, failed_count=0, outcome_unknown_count=0, skipped_count=queue_result.skipped_count + queue_result.blocked_count, dry_run=True, results=[queue_result.as_dict()], ) jobs = ( session.query(CampaignJob) .filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_version_id == version.id, CampaignJob.queue_status == JobQueueStatus.QUEUED.value, CampaignJob.send_status.in_([JobSendStatus.QUEUED.value, JobSendStatus.FAILED_TEMPORARY.value]), ) .order_by(CampaignJob.entry_index.asc()) .all() ) results: list[dict[str, Any]] = [] sent_count = 0 failed_count = 0 outcome_unknown_count = 0 skipped_after_queue = 0 for job in jobs: try: result = send_campaign_job( session, job_id=job.id, dry_run=False, use_rate_limit=use_rate_limit, enqueue_imap_task=enqueue_imap_task, ) result_dict = result.as_dict() results.append(result_dict) if result.status in {JobSendStatus.SMTP_ACCEPTED.value, "already_accepted"}: sent_count += 1 elif result.status == JobSendStatus.OUTCOME_UNKNOWN.value: outcome_unknown_count += 1 else: skipped_after_queue += 1 except Exception as exc: # keep sending other jobs and return per-job details failed_count += 1 results.append({"job_id": job.id, "status": "failed", "message": str(exc)}) return SendCampaignNowResult( campaign_id=campaign.id, version_id=version.id, attempted_count=len(jobs), sent_count=sent_count, failed_count=failed_count, outcome_unknown_count=outcome_unknown_count, skipped_count=queue_result.skipped_count + queue_result.blocked_count + skipped_after_queue, dry_run=False, results=results, ) def enqueue_existing_queued_jobs(session: Session, *, tenant_id: str, campaign_id: str) -> int: if not _celery_enabled(): return 0 campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) jobs = ( session.query(CampaignJob) .filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id, CampaignJob.queue_status == JobQueueStatus.QUEUED.value, CampaignJob.send_status.in_([JobSendStatus.QUEUED.value, JobSendStatus.FAILED_TEMPORARY.value]), ) .order_by(CampaignJob.entry_index.asc()) .all() ) for job in jobs: _celery_enqueue_send_job(job.id) return len(jobs) def pause_campaign_jobs(session: Session, *, tenant_id: str, campaign_id: str) -> dict[str, Any]: campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) changed = ( session.query(CampaignJob) .filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id, CampaignJob.queue_status == JobQueueStatus.QUEUED.value, ) .update({CampaignJob.queue_status: JobQueueStatus.PAUSED.value}, synchronize_session=False) ) if changed: campaign.status = CampaignStatus.READY_TO_QUEUE.value session.add(campaign) session.commit() return {"campaign_id": campaign.id, "paused_count": int(changed)} def resume_campaign_jobs(session: Session, *, tenant_id: str, campaign_id: str, enqueue_celery: bool = True) -> dict[str, Any]: campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) jobs = ( session.query(CampaignJob) .filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id, CampaignJob.queue_status == JobQueueStatus.PAUSED.value, ) .order_by(CampaignJob.entry_index.asc()) .all() ) for job in jobs: job.queue_status = JobQueueStatus.QUEUED.value job.send_status = JobSendStatus.QUEUED.value session.add(job) if jobs: campaign.status = CampaignStatus.QUEUED.value session.add(campaign) session.commit() enqueued_count = 0 if _should_enqueue_celery(enqueue_celery): for job in jobs: _celery_enqueue_send_job(job.id) enqueued_count += 1 return {"campaign_id": campaign.id, "resumed_count": len(jobs), "enqueued_count": enqueued_count} def cancel_campaign_jobs(session: Session, *, tenant_id: str, campaign_id: str) -> dict[str, Any]: campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) jobs = ( session.query(CampaignJob) .filter(CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id) .all() ) cancelled_count = 0 protected_count = 0 version_ids: set[str] = set() for job in jobs: version_ids.add(job.campaign_version_id) if job.send_status in SMTP_ACCEPTED_STATUSES | { JobSendStatus.OUTCOME_UNKNOWN.value, JobSendStatus.CLAIMED.value, JobSendStatus.SENDING.value, }: protected_count += 1 continue if job.send_status != JobSendStatus.CANCELLED.value: job.queue_status = JobQueueStatus.CANCELLED.value job.send_status = JobSendStatus.CANCELLED.value job.claim_token = None session.add(job) cancelled_count += 1 for version_id in version_ids: _update_campaign_after_job(session, campaign.id, version_id) session.commit() return { "campaign_id": campaign.id, "cancelled_count": cancelled_count, "protected_count": protected_count, "campaign_status": campaign.status, } def _eligible_jobs_for_explicit_action( session: Session, *, tenant_id: str, campaign: Campaign, version: CampaignVersion, job_ids: list[str] | None, ) -> list[CampaignJob]: query = session.query(CampaignJob).filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id, CampaignJob.campaign_version_id == version.id, ) if job_ids: query = query.filter(CampaignJob.id.in_(job_ids)) return query.order_by(CampaignJob.entry_index.asc()).all() def queue_failed_jobs_for_retry( session: Session, *, tenant_id: str, campaign_id: str, version_id: str | None = None, job_ids: list[str] | None = None, include_permanent: bool = False, force_max_attempts: bool = False, enqueue_celery: bool = True, dry_run: bool = False, ) -> dict[str, Any]: """Explicitly queue failed jobs; never includes uncertain or accepted jobs.""" campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) version = _get_version_for_campaign(session, campaign, version_id=version_id) _ensure_version_validated_and_locked(version) snapshot = ensure_execution_snapshot(session, version) allowed = {JobSendStatus.FAILED_TEMPORARY.value} if include_permanent: allowed.add(JobSendStatus.FAILED_PERMANENT.value) selected: list[CampaignJob] = [] skipped: list[dict[str, str]] = [] for job in _eligible_jobs_for_explicit_action( session, tenant_id=tenant_id, campaign=campaign, version=version, job_ids=job_ids, ): if job.send_status not in allowed: skipped.append({"job_id": job.id, "reason": f"status {job.send_status} is not an explicit retry candidate"}) continue if not force_max_attempts and job.attempt_count >= snapshot.delivery.retry.max_attempts: skipped.append({"job_id": job.id, "reason": "configured maximum attempts reached"}) continue selected.append(job) if not dry_run: job.queue_status = JobQueueStatus.QUEUED.value job.send_status = JobSendStatus.QUEUED.value job.queued_at = _utcnow() job.claimed_at = None job.claim_token = None job.smtp_started_at = None job.outcome_unknown_at = None session.add(job) if not dry_run: if selected: campaign.status = CampaignStatus.QUEUED.value version.workflow_state = CampaignVersionWorkflowState.QUEUED.value session.add(campaign) session.add(version) session.commit() enqueued = 0 if _should_enqueue_celery(enqueue_celery) and not dry_run: for job in selected: _celery_enqueue_send_job(job.id) enqueued += 1 return { "campaign_id": campaign.id, "version_id": version.id, "action": "retry_failed", "selected_count": len(selected), "enqueued_count": enqueued, "skipped": skipped, "dry_run": dry_run, } def queue_unattempted_jobs( session: Session, *, tenant_id: str, campaign_id: str, version_id: str | None = None, job_ids: list[str] | None = None, enqueue_celery: bool = True, dry_run: bool = False, ) -> dict[str, Any]: """Explicitly queue built jobs that have never started an SMTP attempt.""" campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) version = _get_version_for_campaign(session, campaign, version_id=version_id) _ensure_version_validated_and_locked(version) ensure_execution_snapshot(session, version) selected: list[CampaignJob] = [] skipped: list[dict[str, str]] = [] for job in _eligible_jobs_for_explicit_action( session, tenant_id=tenant_id, campaign=campaign, version=version, job_ids=job_ids, ): eligible = ( job.attempt_count == 0 and job.send_status in {JobSendStatus.NOT_QUEUED.value, JobSendStatus.CANCELLED.value} and job.build_status == JobBuildStatus.BUILT.value and job.validation_status in QUEUEABLE_VALIDATION_STATUSES ) if not eligible: skipped.append({"job_id": job.id, "reason": "job is not an unattempted queueable message"}) continue selected.append(job) if not dry_run: job.queue_status = JobQueueStatus.QUEUED.value job.send_status = JobSendStatus.QUEUED.value job.queued_at = _utcnow() job.claimed_at = None job.claim_token = None job.smtp_started_at = None job.outcome_unknown_at = None job.last_error = None session.add(job) if not dry_run: if selected: campaign.status = CampaignStatus.QUEUED.value version.workflow_state = CampaignVersionWorkflowState.QUEUED.value session.add(campaign) session.add(version) session.commit() enqueued = 0 if _should_enqueue_celery(enqueue_celery) and not dry_run: for job in selected: _celery_enqueue_send_job(job.id) enqueued += 1 return { "campaign_id": campaign.id, "version_id": version.id, "action": "send_unattempted", "selected_count": len(selected), "enqueued_count": enqueued, "skipped": skipped, "dry_run": dry_run, } def reconcile_job_outcome( session: Session, *, tenant_id: str, campaign_id: str, job_id: str, decision: str, note: str | None = None, ) -> dict[str, Any]: """Record an operator decision for an uncertain/incomplete worker state. ``smtp_accepted`` protects the message from retry and enables IMAP append. ``not_sent`` converts it into a failed-temporary candidate; retry remains a separate explicit action. """ campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) job = session.get(CampaignJob, job_id) if not job or job.tenant_id != tenant_id or job.campaign_id != campaign.id: raise QueueingError("Campaign job not found or not accessible") if job.send_status not in { JobSendStatus.OUTCOME_UNKNOWN.value, JobSendStatus.SENDING.value, JobSendStatus.CLAIMED.value, }: raise QueueingError(f"Job status {job.send_status} does not require reconciliation") version = _get_version_for_campaign(session, campaign, version_id=job.campaign_version_id) snapshot = ensure_execution_snapshot(session, version) now = _utcnow() attempt = _unfinished_attempt(session, job) if decision == "smtp_accepted": job.send_status = JobSendStatus.SMTP_ACCEPTED.value job.queue_status = JobQueueStatus.DRAFT.value job.sent_at = job.sent_at or now job.outcome_unknown_at = None job.claim_token = None job.last_error = note or "Operator confirmed SMTP acceptance after an uncertain outcome." job.imap_status = JobImapStatus.PENDING.value if snapshot.delivery.imap_append_sent.enabled else JobImapStatus.NOT_REQUESTED.value files_integration().mark_job_attachment_uses_sent(session, job) attempt_status = "reconciled_smtp_accepted" elif decision == "not_sent": job.send_status = JobSendStatus.FAILED_TEMPORARY.value job.queue_status = JobQueueStatus.DRAFT.value job.outcome_unknown_at = None job.claim_token = None job.last_error = note or "Operator confirmed that SMTP did not accept this message." attempt_status = "reconciled_not_sent" else: raise QueueingError("decision must be 'smtp_accepted' or 'not_sent'") if attempt: attempt.status = attempt_status attempt.finished_at = now attempt.error_type = "OperatorReconciliation" attempt.error_message = job.last_error session.add(attempt) session.add(job) _update_campaign_after_job(session, campaign.id, version.id) session.commit() return { "campaign_id": campaign.id, "version_id": version.id, "job_id": job.id, "decision": decision, "send_status": job.send_status, "imap_status": job.imap_status, "note": job.last_error, } def _verify_eml_evidence(job: CampaignJob, payload: bytes) -> None: if job.eml_size_bytes is not None and len(payload) != job.eml_size_bytes: raise SendJobError( f"Generated EML size mismatch for job {job.id}: expected {job.eml_size_bytes}, got {len(payload)}" ) if job.eml_sha256: actual = hashlib.sha256(payload).hexdigest() if actual != job.eml_sha256: raise SendJobError( f"Generated EML SHA-256 mismatch for job {job.id}; rebuild the campaign version before delivery" ) if job.message_id_header: parsed = BytesParser(policy=policy.default).parsebytes(payload) message_id = parsed.get("Message-ID") parsed_message_id = str(message_id) if message_id else None if parsed_message_id != job.message_id_header: raise SendJobError( f"Generated EML Message-ID mismatch for job {job.id}; rebuild the campaign version before delivery" ) def _load_eml_bytes_for_job(job: CampaignJob) -> bytes: if job.eml_local_path: path = Path(job.eml_local_path) if not path.exists(): raise SendJobError(f"Generated EML file does not exist: {path}") payload = path.read_bytes() _verify_eml_evidence(job, payload) return payload raise SendJobError("Only local EML paths are supported for sending in this implementation step") def _load_eml_for_job(job: CampaignJob): return BytesParser(policy=policy.default).parsebytes(_load_eml_bytes_for_job(job)) def _addresses_from_job(job: CampaignJob, field: str) -> list[str]: data = job.resolved_recipients or {} values = data.get(field) or [] return [item.get("email") for item in values if isinstance(item, dict) and item.get("email")] def _sender_from_job(job: CampaignJob, snapshot: ExecutionSnapshot) -> str: data = job.resolved_recipients or {} bounce_to = _addresses_from_job(job, "bounce_to") if bounce_to: return bounce_to[0] from_data = data.get("from") if isinstance(data, dict) else None if isinstance(from_data, dict) and from_data.get("email"): return from_data["email"] if snapshot.smtp.username: return snapshot.smtp.username raise SmtpConfigurationError("No envelope sender could be determined") def _from_header_from_job(job: CampaignJob, snapshot: ExecutionSnapshot) -> str | None: data = job.resolved_recipients or {} from_data = data.get("from") if isinstance(data, dict) else None if isinstance(from_data, dict) and from_data.get("email"): return str(from_data["email"]) return snapshot.smtp.username def _recipients_from_job(job: CampaignJob) -> list[str]: recipients: list[str] = [] for field in ["to", "cc", "bcc"]: recipients.extend(_addresses_from_job(job, field)) # Preserve order while de-duplicating. return list(dict.fromkeys(recipients)) def _claim_job_for_sending(session: Session, job: CampaignJob) -> str | None: """Atomically claim a queued job and return the claim token. A duplicate task can observe CLAIMED/SENDING but cannot acquire a second claim. CLAIMED is deliberately not timed out automatically: an operator can safely release it after checking worker state, while an automatic timeout could race a slow worker. """ claim_token = str(uuid4()) changed = ( session.query(CampaignJob) .filter( CampaignJob.id == job.id, CampaignJob.queue_status == JobQueueStatus.QUEUED.value, CampaignJob.send_status == JobSendStatus.QUEUED.value, ) .update( { CampaignJob.queue_status: JobQueueStatus.SENDING.value, CampaignJob.send_status: JobSendStatus.CLAIMED.value, CampaignJob.claimed_at: _utcnow(), CampaignJob.claim_token: claim_token, CampaignJob.last_error: None, }, synchronize_session=False, ) ) session.commit() session.expire_all() return claim_token if changed == 1 else None def _record_attempt_start(session: Session, job: CampaignJob, claim_token: str) -> SendAttempt: now = _utcnow() attempt = SendAttempt( job_id=job.id, attempt_number=job.attempt_count + 1, status="smtp_in_progress", claim_token=claim_token, started_at=now, ) job.attempt_count += 1 job.queue_status = JobQueueStatus.SENDING.value job.send_status = JobSendStatus.SENDING.value job.smtp_started_at = now job.outcome_unknown_at = None job.last_error = None session.add(attempt) session.add(job) session.commit() return attempt def _unfinished_attempt(session: Session, job: CampaignJob) -> SendAttempt | None: return ( session.query(SendAttempt) .filter(SendAttempt.job_id == job.id, SendAttempt.finished_at.is_(None)) .order_by(SendAttempt.attempt_number.desc()) .first() ) def mark_job_outcome_unknown(session: Session, job: CampaignJob, *, reason: str) -> SendJobResult: """Freeze an uncertain SMTP result and prohibit automatic redelivery.""" now = _utcnow() attempt = _unfinished_attempt(session, job) if attempt: attempt.status = "outcome_unknown" attempt.finished_at = now attempt.error_type = "OutcomeUnknown" attempt.error_message = reason session.add(attempt) job.queue_status = JobQueueStatus.DRAFT.value job.send_status = JobSendStatus.OUTCOME_UNKNOWN.value job.outcome_unknown_at = now job.last_error = reason session.add(job) _update_campaign_after_job(session, job.campaign_id, job.campaign_version_id) session.commit() return SendJobResult( job_id=job.id, status=JobSendStatus.OUTCOME_UNKNOWN.value, attempt_number=job.attempt_count, message=reason, ) def _update_campaign_after_job(session: Session, campaign_id: str, version_id: str | None = None) -> None: session.flush() campaign = session.get(Campaign, campaign_id) if not campaign: return base_filters = [CampaignJob.campaign_id == campaign_id] if version_id: base_filters.append(CampaignJob.campaign_version_id == version_id) counts = { status: session.query(CampaignJob).filter(*base_filters, CampaignJob.send_status == status).count() for status in {item.value for item in JobSendStatus} } accepted = sum(counts.get(status, 0) for status in SMTP_ACCEPTED_STATUSES) unknown = counts.get(JobSendStatus.OUTCOME_UNKNOWN.value, 0) failed = counts.get(JobSendStatus.FAILED_TEMPORARY.value, 0) + counts.get(JobSendStatus.FAILED_PERMANENT.value, 0) active = ( counts.get(JobSendStatus.QUEUED.value, 0) + counts.get(JobSendStatus.CLAIMED.value, 0) + counts.get(JobSendStatus.SENDING.value, 0) ) cancelled = counts.get(JobSendStatus.CANCELLED.value, 0) # Blocked/excluded build rows are reportable but are not delivery attempts. # Only a built, queueable message counts as an unattempted part of an # execution when deriving a partial outcome. not_started = ( session.query(CampaignJob) .filter( *base_filters, CampaignJob.send_status == JobSendStatus.NOT_QUEUED.value, CampaignJob.build_status == JobBuildStatus.BUILT.value, CampaignJob.validation_status.in_(list(QUEUEABLE_VALIDATION_STATUSES)), ) .count() ) version = session.get(CampaignVersion, version_id) if version_id else None if active: campaign.status = CampaignStatus.SENDING.value if version: version.workflow_state = CampaignVersionWorkflowState.SENDING.value elif accepted and (failed or unknown or cancelled or not_started): campaign.status = CampaignStatus.PARTIALLY_COMPLETED.value if version: version.workflow_state = CampaignVersionWorkflowState.PARTIALLY_COMPLETED.value elif unknown: campaign.status = CampaignStatus.OUTCOME_UNKNOWN.value if version: version.workflow_state = CampaignVersionWorkflowState.OUTCOME_UNKNOWN.value elif failed: campaign.status = CampaignStatus.FAILED.value if version: version.workflow_state = CampaignVersionWorkflowState.FAILED.value elif accepted: campaign.status = CampaignStatus.SENT.value if version: version.workflow_state = CampaignVersionWorkflowState.COMPLETED.value elif cancelled: campaign.status = CampaignStatus.CANCELLED.value if version: version.workflow_state = CampaignVersionWorkflowState.CANCELLED.value if version and (accepted or unknown): if version.locked_at is None: version.locked_at = _utcnow() session.add(version) session.add(campaign) def send_campaign_job( session: Session, *, job_id: str, dry_run: bool = False, use_rate_limit: bool = True, enqueue_imap_task: bool = False, ) -> SendJobResult: job = session.get(CampaignJob, job_id) if not job: raise SendJobError(f"Job not found: {job_id}") if job.queue_status == JobQueueStatus.CANCELLED.value or job.send_status == JobSendStatus.CANCELLED.value: return SendJobResult(job_id=job_id, status="cancelled", attempt_number=job.attempt_count, dry_run=dry_run) if job.queue_status == JobQueueStatus.PAUSED.value: return SendJobResult(job_id=job_id, status="paused", attempt_number=job.attempt_count, dry_run=dry_run) if job.send_status in SMTP_ACCEPTED_STATUSES: return SendJobResult(job_id=job_id, status="already_accepted", attempt_number=job.attempt_count, dry_run=dry_run) if job.send_status == JobSendStatus.OUTCOME_UNKNOWN.value: return SendJobResult( job_id=job_id, status=JobSendStatus.OUTCOME_UNKNOWN.value, attempt_number=job.attempt_count, dry_run=dry_run, message="SMTP outcome is unresolved; reconcile it before any retry.", ) if job.send_status == JobSendStatus.SENDING.value: return mark_job_outcome_unknown( session, job, reason="A delivery task resumed while the previous SMTP attempt was still marked in progress. Automatic resend was stopped.", ) if job.send_status == JobSendStatus.CLAIMED.value: return SendJobResult( job_id=job_id, status="already_claimed", attempt_number=job.attempt_count, dry_run=dry_run, message="Another worker has claimed this job, or a stale pre-SMTP claim requires operator review.", ) if job.queue_status != JobQueueStatus.QUEUED.value or job.send_status != JobSendStatus.QUEUED.value: raise SendJobError(f"Job is not explicitly queued for delivery: queue={job.queue_status}, send={job.send_status}") version = session.get(CampaignVersion, job.campaign_version_id) if not version: raise SendJobError("Campaign version not found") try: snapshot = ensure_execution_snapshot(session, version) except ExecutionSnapshotError as exc: raise SendJobError(str(exc)) from exc message_bytes = _load_eml_bytes_for_job(job) envelope_from = _sender_from_job(job, snapshot) envelope_recipients = _recipients_from_job(job) if not envelope_recipients: raise SmtpConfigurationError("No envelope recipients could be determined") if dry_run: return SendJobResult( job_id=job.id, status="dry_run", attempt_number=job.attempt_count, dry_run=True, message=f"Would send to {len(envelope_recipients)} recipient(s) from {envelope_from}", ) claim_token = _claim_job_for_sending(session, job) if claim_token is None: current = session.get(CampaignJob, job.id) if not current: raise SendJobError(f"Job disappeared while claiming: {job.id}") if current.send_status == JobSendStatus.SENDING.value: return mark_job_outcome_unknown( session, current, reason="A duplicate/redelivered task found an unfinished SMTP attempt. Automatic resend was stopped.", ) return SendJobResult( job_id=current.id, status="not_claimed", attempt_number=current.attempt_count, message=f"Job is no longer queueable: {current.send_status}", ) job = session.get(CampaignJob, job.id) assert job is not None mail_integration().wait_for_rate_limit( key=f"tenant:{job.tenant_id}:campaign:{job.campaign_id}", messages_per_minute=snapshot.delivery.rate_limit.messages_per_minute, enabled=use_rate_limit, ) attempt = _record_attempt_start(session, job, claim_token) try: smtp_config = runtime_smtp_config(session, version, snapshot) mail_integration().assert_mail_policy_allows_send( session, tenant_id=job.tenant_id, campaign_id=job.campaign_id, smtp=smtp_config, envelope_sender=envelope_from, from_header=_from_header_from_job(job, snapshot), recipients=envelope_recipients, ) result = mail_integration().send_email_bytes( message_bytes, smtp_config=smtp_config, envelope_from=envelope_from, envelope_recipients=envelope_recipients, ) if result.accepted_count <= 0: raise SmtpSendError("SMTP did not accept any envelope recipients", temporary=False) refused_warning = None if result.refused_recipients: refused_warning = ( f"SMTP accepted {result.accepted_count}/{len(result.envelope_recipients)} envelope recipient(s); " f"refused recipients: {json.dumps(result.refused_recipients, default=str, sort_keys=True)}" ) attempt.finished_at = _utcnow() attempt.status = "smtp_accepted_with_refusals" if refused_warning else JobSendStatus.SMTP_ACCEPTED.value attempt.smtp_response = json.dumps(asdict(result), default=str) job.queue_status = JobQueueStatus.DRAFT.value job.send_status = JobSendStatus.SMTP_ACCEPTED.value job.sent_at = _utcnow() job.claim_token = None job.outcome_unknown_at = None if snapshot.delivery.imap_append_sent.enabled: job.imap_status = JobImapStatus.PENDING.value else: job.imap_status = JobImapStatus.NOT_REQUESTED.value job.last_error = refused_warning files_integration().mark_job_attachment_uses_sent(session, job) session.add(attempt) session.add(job) _update_campaign_after_job(session, job.campaign_id, job.campaign_version_id) session.commit() if enqueue_imap_task and _celery_enabled() and job.imap_status == JobImapStatus.PENDING.value: _celery_enqueue_append_sent_job(job.id) return SendJobResult( job_id=job.id, status=JobSendStatus.SMTP_ACCEPTED.value, attempt_number=attempt.attempt_number, message=refused_warning, ) except SmtpSendError as exc: if getattr(exc, "outcome_unknown", False): attempt.status = JobSendStatus.OUTCOME_UNKNOWN.value attempt.finished_at = _utcnow() attempt.error_type = exc.__class__.__name__ attempt.error_message = str(exc) session.add(attempt) job.claim_token = None session.add(job) session.flush() return mark_job_outcome_unknown(session, job, reason=str(exc)) attempt.finished_at = _utcnow() attempt.error_type = exc.__class__.__name__ attempt.error_message = str(exc) retryable = bool(getattr(exc, "temporary", False)) job.last_error = str(exc) job.queue_status = JobQueueStatus.DRAFT.value job.send_status = JobSendStatus.FAILED_TEMPORARY.value if retryable else JobSendStatus.FAILED_PERMANENT.value job.claim_token = None attempt.status = job.send_status session.add(attempt) session.add(job) _update_campaign_after_job(session, job.campaign_id, job.campaign_version_id) session.commit() raise except (MailProfileError, SmtpConfigurationError, SendJobError, OSError) as exc: attempt.finished_at = _utcnow() attempt.error_type = exc.__class__.__name__ attempt.error_message = str(exc) attempt.status = JobSendStatus.FAILED_PERMANENT.value job.last_error = str(exc) job.queue_status = JobQueueStatus.DRAFT.value job.send_status = JobSendStatus.FAILED_PERMANENT.value job.claim_token = None session.add(attempt) session.add(job) _update_campaign_after_job(session, job.campaign_id, job.campaign_version_id) session.commit() raise def _record_imap_attempt_start(session: Session, job: CampaignJob) -> ImapAppendAttempt: existing_count = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count() attempt = ImapAppendAttempt( job_id=job.id, attempt_number=existing_count + 1, status="running", ) job.imap_status = JobImapStatus.PENDING.value job.last_error = None session.add(attempt) session.add(job) session.commit() return attempt def _effective_imap_folder(snapshot: ExecutionSnapshot) -> str: delivery_folder = snapshot.delivery.imap_append_sent.folder if delivery_folder and delivery_folder != "auto": return delivery_folder if snapshot.imap and snapshot.imap.sent_folder and snapshot.imap.sent_folder != "auto": return snapshot.imap.sent_folder return "auto" def append_sent_for_job(session: Session, *, job_id: str, dry_run: bool = False) -> AppendSentResult: """Append one successfully sent job's exact EML to the configured IMAP Sent folder.""" job = session.get(CampaignJob, job_id) if not job: raise SendJobError(f"Job not found: {job_id}") if job.send_status not in SMTP_ACCEPTED_STATUSES: return AppendSentResult(job_id=job_id, status="not_sent", attempt_number=0, dry_run=dry_run, message="SMTP has not accepted this job") if job.imap_status == JobImapStatus.NOT_REQUESTED.value: return AppendSentResult(job_id=job_id, status="not_requested", attempt_number=0, dry_run=dry_run) if job.imap_status == JobImapStatus.APPENDED.value: attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count() return AppendSentResult(job_id=job_id, status="already_appended", attempt_number=attempts, dry_run=dry_run) if job.imap_status == JobImapStatus.SKIPPED.value: attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count() return AppendSentResult(job_id=job_id, status="skipped", attempt_number=attempts, dry_run=dry_run) version = session.get(CampaignVersion, job.campaign_version_id) if not version: raise SendJobError("Campaign version not found") try: snapshot = ensure_execution_snapshot(session, version) except ExecutionSnapshotError as exc: raise SendJobError(str(exc)) from exc if not snapshot.delivery.imap_append_sent.enabled: job.imap_status = JobImapStatus.NOT_REQUESTED.value session.add(job) session.commit() return AppendSentResult(job_id=job.id, status="not_requested", attempt_number=0, dry_run=dry_run) imap_config = runtime_imap_config(session, version, snapshot) if not imap_config: job.imap_status = JobImapStatus.SKIPPED.value job.last_error = "IMAP append requested, but the execution snapshot has no IMAP configuration" session.add(job) session.commit() return AppendSentResult(job_id=job.id, status="skipped", attempt_number=0, dry_run=dry_run, message=job.last_error) message_bytes = _load_eml_bytes_for_job(job) folder = _effective_imap_folder(snapshot) if dry_run: attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count() return AppendSentResult( job_id=job.id, status="dry_run", attempt_number=attempts, dry_run=True, folder=folder, message=f"Would append {len(message_bytes)} bytes to IMAP folder {folder!r}", ) attempt = _record_imap_attempt_start(session, job) try: mail_integration().assert_mail_policy_allows_send( session, tenant_id=job.tenant_id, campaign_id=job.campaign_id, smtp=snapshot.smtp, imap=imap_config, ) result = mail_integration().append_message_to_sent( message_bytes, imap_config=imap_config, folder=None if folder == "auto" else folder, ) attempt.status = "appended" attempt.folder = result.folder job.imap_status = JobImapStatus.APPENDED.value job.last_error = None files_integration().mark_job_attachment_uses_sent(session, job) session.add(attempt) session.add(job) session.commit() return AppendSentResult(job_id=job.id, status="appended", attempt_number=attempt.attempt_number, folder=result.folder) except (MailProfileError, ImapConfigurationError, ImapAppendError, SendJobError, OSError) as exc: attempt.status = "failed" attempt.folder = None if folder == "auto" else folder attempt.error_message = str(exc) job.imap_status = JobImapStatus.FAILED.value job.last_error = str(exc) session.add(attempt) session.add(job) session.commit() raise def enqueue_pending_imap_appends( session: Session, *, tenant_id: str, campaign_id: str, enqueue_celery: bool = True, run_inline: bool = False, dry_run: bool = False, ) -> dict[str, Any]: campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id) jobs = ( session.query(CampaignJob) .filter( CampaignJob.tenant_id == tenant_id, CampaignJob.campaign_id == campaign.id, CampaignJob.send_status.in_(list(SMTP_ACCEPTED_STATUSES)), CampaignJob.imap_status.in_([JobImapStatus.PENDING.value, JobImapStatus.FAILED.value]), ) .order_by(CampaignJob.entry_index.asc()) .all() ) should_enqueue = _should_enqueue_celery(enqueue_celery) and not dry_run and not run_inline results: list[dict[str, Any]] = [] appended_count = 0 failed_count = 0 skipped_count = 0 if run_inline or dry_run: for job in jobs: try: result = append_sent_for_job(session, job_id=job.id, dry_run=dry_run) payload = result.as_dict() results.append(payload) if result.status == JobImapStatus.APPENDED.value: appended_count += 1 elif result.status in {"skipped", "not_requested", "not_sent", "already_appended", "dry_run"}: skipped_count += 1 except Exception as exc: # keep processing later jobs and expose per-job details failed_count += 1 results.append({"job_id": job.id, "status": "failed", "message": str(exc)}) elif should_enqueue: for job in jobs: _celery_enqueue_append_sent_job(job.id) return { "campaign_id": campaign.id, "pending_count": len(jobs), "enqueued_count": len(jobs) if should_enqueue else 0, "processed_count": len(results) if run_inline and not dry_run else 0, "appended_count": appended_count, "failed_count": failed_count, "skipped_count": skipped_count, "dry_run": dry_run, "run_inline": run_inline, "results": results, } def next_retry_delay(snapshot: ExecutionSnapshot, attempt_count: int) -> int: delays = snapshot.delivery.retry.backoff_seconds or [60] index = max(0, min(attempt_count - 1, len(delays) - 1)) return int(delays[index])