GovOPlaN/govoplan-campaign
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: pause campaign batches on SMTP failure

Browse Source
This commit is contained in:
zemion
2026-08-20 17:39:08 +02:00
parent 69e6588a89
commit c846c249b8
12 changed files with 385 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_campaign/backend/sending/jobs.py
+182 -27
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
import json
from collections import Counter
from contextlib import nullcontext
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from email import policy
@@ -225,6 +226,11 @@ class SendCampaignNowResult:
failed_count: int
outcome_unknown_count: int
skipped_count: int
paused_count: int = 0
batch_state: str = "not_started"
batch_pause_reason_code: str | None = None
smtp_connection_count: int = 0
smtp_reconnect_count: int = 0
preflight_count: int = 0
synchronous_send_policy: dict[str, Any] | None = None
dry_run: bool = False
@@ -239,7 +245,12 @@ class SendCampaignNowResult:
"failed_count": self.failed_count,
"outcome_unknown_count": self.outcome_unknown_count,
"skipped_count": self.skipped_count,
"paused_count": self.paused_count,
"preflight_count": self.preflight_count,
"batch_state": self.batch_state,
"batch_pause_reason_code": self.batch_pause_reason_code,
"smtp_connection_count": self.smtp_connection_count,
"smtp_reconnect_count": self.smtp_reconnect_count,
"delivery_mode": "synchronous",
"synchronous_send_policy": self.synchronous_send_policy or {},
"dry_run": self.dry_run,
@@ -1070,48 +1081,91 @@ def send_campaign_now(
jobs=jobs,
policy=synchronous_policy,
)
# Queue state and its inbox notification become durable only after every
# message and the selected transport revision have passed preflight. This
# preserves late-ack recovery without leaving rejected work eligible for a
# background worker.
session.commit()
results: list[dict[str, Any]] = []
sent_count = 0
failed_count = 0
outcome_unknown_count = 0
skipped_after_queue = 0
for job in jobs:
try:
result = _deliver_job_with_recovery(
session,
job=job,
context=delivery_contexts[job.id],
use_rate_limit=use_rate_limit,
enqueue_imap_task=enqueue_imap_task,
)
result_dict = result.as_dict()
results.append(result_dict)
if result.status in DELIVERY_ACCEPTED_STATUSES | {"already_accepted"}:
sent_count += 1
elif result.status == JobSendStatus.OUTCOME_UNKNOWN.value:
outcome_unknown_count += 1
else:
skipped_after_queue += 1
except Exception as exc: # keep sending other jobs and return per-job details
failed_count += 1
results.append({"job_id": job.id, "status": "failed", "message": str(exc)})
attempted_count = 0
paused_count = 0
pause_reason_code: str | None = None
batch_state = "ready"
batch_manager = _synchronous_smtp_batch_manager(
session,
jobs=jobs,
contexts=delivery_contexts,
)
try:
with batch_manager as smtp_batch:
# Queue state becomes durable only after local and SMTP
# DNS/connectivity/TLS/auth preflight succeeds.
session.commit()
for index, job in enumerate(jobs):
attempted_count += 1
try:
result = _deliver_job_with_recovery(
session,
job=job,
context=delivery_contexts[job.id],
use_rate_limit=use_rate_limit,
enqueue_imap_task=enqueue_imap_task,
)
result_dict = result.as_dict()
results.append(result_dict)
if result.status in DELIVERY_ACCEPTED_STATUSES | {"already_accepted"}:
sent_count += 1
elif result.status == JobSendStatus.OUTCOME_UNKNOWN.value:
outcome_unknown_count += 1
else:
skipped_after_queue += 1
except Exception as exc:
failed_count += 1
results.append({"job_id": job.id, "status": "failed", "message": str(exc)})
if isinstance(exc, SmtpSendError) and exc.systemic:
pause_reason_code = exc.reason_code or "smtp_systemic_failure"
paused_count = _pause_jobs_after_systemic_smtp_failure(
session,
campaign_id=job.campaign_id,
exclude_job_id=job.id,
reason_code=pause_reason_code,
)
batch_state = "paused"
for remaining in jobs[index + 1 :]:
results.append(
{
"job_id": remaining.id,
"status": "paused",
"message": "Batch paused after a systemic SMTP failure.",
}
)
break
smtp_connection_count = int(getattr(smtp_batch, "connection_count", 0) or 0)
smtp_reconnect_count = int(getattr(smtp_batch, "reconnect_count", 0) or 0)
except (MailProfileError, SmtpConfigurationError, SmtpSendError, OSError) as exc:
session.rollback()
reason_code = str(getattr(exc, "reason_code", "") or "smtp_batch_preflight_failed")
raise SynchronousSendRejected(
"SMTP batch preflight could not validate DNS, connectivity, TLS, and authentication; no message was sent.",
reason=reason_code,
eligible_count=len(jobs),
policy=synchronous_policy,
) from exc
return SendCampaignNowResult(
campaign_id=campaign.id,
version_id=version.id,
attempted_count=len(jobs),
attempted_count=attempted_count,
sent_count=sent_count,
failed_count=failed_count,
outcome_unknown_count=outcome_unknown_count,
skipped_count=queue_result.skipped_count
+ queue_result.blocked_count
+ skipped_after_queue,
paused_count=paused_count,
batch_state=batch_state,
batch_pause_reason_code=pause_reason_code,
smtp_connection_count=smtp_connection_count,
smtp_reconnect_count=smtp_reconnect_count,
preflight_count=len(delivery_contexts),
synchronous_send_policy=synchronous_policy.as_dict(),
dry_run=False,
@@ -1193,6 +1247,100 @@ def _preflight_synchronous_send_batch(
return contexts
def _synchronous_smtp_batch_manager(
session: Session,
*,
jobs: list[CampaignJob],
contexts: dict[str, _SendJobDeliveryContext],
):
mail_items = [
(job, contexts[job.id])
for job in jobs
if DeliveryChannelPolicy(job.delivery_channel_policy).uses_mail
]
if not mail_items:
return nullcontext(None)
first_job, first_context = mail_items[0]
envelope_froms = {str(context.envelope_from or "") for _job, context in mail_items}
transport_keys = {
(
context.snapshot.mail_profile_id,
context.snapshot.smtp_transport_revision,
context.snapshot.smtp_server_id,
context.snapshot.smtp_credential_id,
)
for _job, context in mail_items
}
if len(envelope_froms) != 1 or "" in envelope_froms or len(transport_keys) != 1:
raise SynchronousSendRejected(
"A synchronous SMTP batch requires one frozen sender and transport selection.",
reason="smtp_batch_transport_mismatch",
eligible_count=len(jobs),
)
recipients = sorted(
{
recipient
for _job, context in mail_items
for recipient in context.envelope_recipients
}
)
return mail_integration().campaign_smtp_batch(
session,
tenant_id=first_job.tenant_id,
campaign_id=first_job.campaign_id,
profile_id=first_context.snapshot.mail_profile_id,
envelope_from=str(first_context.envelope_from),
envelope_recipients=recipients,
from_header=_from_header_from_job(first_job),
expected_smtp_transport_revision=first_context.snapshot.smtp_transport_revision or "",
smtp_server_id=first_context.snapshot.smtp_server_id,
smtp_credential_id=first_context.snapshot.smtp_credential_id,
)
def _pause_jobs_after_systemic_smtp_failure(
session: Session,
*,
campaign_id: str,
exclude_job_id: str,
reason_code: str,
) -> int:
reason = f"SMTP batch paused after systemic failure ({reason_code[:80]})."
changed = (
session.query(CampaignJob)
.filter(
CampaignJob.campaign_id == campaign_id,
CampaignJob.id != exclude_job_id,
CampaignJob.queue_status == JobQueueStatus.QUEUED.value,
CampaignJob.send_status.in_(
[JobSendStatus.QUEUED.value, JobSendStatus.FAILED_TEMPORARY.value]
),
)
.update(
{
CampaignJob.queue_status: JobQueueStatus.PAUSED.value,
CampaignJob.last_error: reason,
},
synchronize_session=False,
)
)
campaign = session.get(Campaign, campaign_id)
if changed and campaign is not None:
campaign.status = CampaignStatus.READY_TO_QUEUE.value
session.add(campaign)
audit_event(
session,
tenant_id=campaign.tenant_id if campaign is not None else None,
user_id=None,
action="campaign.smtp_batch_paused",
object_type="campaign",
object_id=campaign_id,
details={"reason_code": reason_code[:80], "paused_count": int(changed)},
)
session.commit()
return int(changed)
def enqueue_existing_queued_jobs(
session: Session, *, tenant_id: str, campaign_id: str
) -> int:
@@ -3652,6 +3800,13 @@ def _send_claimed_mail_only_job(
outcome_unknown = _record_smtp_send_error(
session, job=job, attempt=attempt, exc=exc
)
if exc.systemic:
_pause_jobs_after_systemic_smtp_failure(
session,
campaign_id=job.campaign_id,
exclude_job_id=job.id,
reason_code=exc.reason_code or "smtp_systemic_failure",
)
if outcome_unknown is not None:
return outcome_unknown
raise