GovOPlaN/govoplan-campaign
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: pause campaign batches on SMTP failure

Browse Source
This commit is contained in:
zemion
2026-08-20 17:39:08 +02:00
parent 69e6588a89
commit c846c249b8
12 changed files with 385 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_campaign/backend/integrations.py
+38 -1
View File
@@ -74,11 +74,21 @@ class SmtpConfigurationError(RuntimeError):
class SmtpSendError(RuntimeError):
def __init__(
self, message: str, *, temporary: bool = False, outcome_unknown: bool = False
self,
message: str,
*,
temporary: bool = False,
outcome_unknown: bool = False,
systemic: bool = False,
reason_code: str | None = None,
phase: str = "send",
) -> None:
super().__init__(message)
self.temporary = temporary
self.outcome_unknown = outcome_unknown
self.systemic = systemic
self.reason_code = reason_code
self.phase = phase
class ImapConfigurationError(RuntimeError):
@@ -298,6 +308,9 @@ class MailCampaignIntegration:
str(exc),
temporary=bool(getattr(exc, "temporary", False)),
outcome_unknown=bool(getattr(exc, "outcome_unknown", False)),
systemic=bool(getattr(exc, "systemic", False)),
reason_code=str(getattr(exc, "reason_code", "") or "") or None,
phase=str(getattr(exc, "phase", "send") or "send"),
) from exc
except getattr(
delegate, "SmtpConfigurationError", SmtpConfigurationError
@@ -306,6 +319,30 @@ class MailCampaignIntegration:
except getattr(delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
@contextmanager
def campaign_smtp_batch(self, *args: Any, **kwargs: Any) -> Iterator[Any]:
delegate = self._require()
method = getattr(delegate, "campaign_smtp_batch", None)
if not callable(method):
yield None
return
try:
with method(*args, **kwargs) as state:
yield state
except getattr(delegate, "SmtpSendError", SmtpSendError) as exc:
raise SmtpSendError(
str(exc),
temporary=bool(getattr(exc, "temporary", False)),
outcome_unknown=bool(getattr(exc, "outcome_unknown", False)),
systemic=bool(getattr(exc, "systemic", False)),
reason_code=str(getattr(exc, "reason_code", "") or "") or None,
phase=str(getattr(exc, "phase", "preflight") or "preflight"),
) from exc
except getattr(delegate, "SmtpConfigurationError", SmtpConfigurationError) as exc:
raise SmtpConfigurationError(str(exc)) from exc
except getattr(delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def append_campaign_message_to_sent(self, *args: Any, **kwargs: Any) -> Any:
delegate = self._require()
try:
src/govoplan_campaign/backend/manifest.py
+1 -1
View File
@@ -913,7 +913,7 @@ manifest = ModuleManifest(
id="campaigns.mail-profile-operations",
title="Operate profile-backed campaign delivery",
summary="Workers re-authorize and resolve Mail profiles at execution time while Campaign retains only opaque Mail-owned revisions and outcomes.",
body="A legacy snapshot, unauthorized or inactive profile, profile-reference mismatch, or changed SMTP/IMAP transport revision stops delivery. Preserve the record, migrate or correct the profile selection, revalidate, rebuild, and only then queue again. Password-only rotation remains possible without copying secrets into Campaign. Uncertain SMTP and IMAP effects remain blocked until an evidence-backed operator reconciliation. If Campaign becomes unavailable to the tenant after a job was accepted, the worker leaves the job untouched and reports an operator action instead of sending or dropping it.",
body="A legacy snapshot, unauthorized or inactive profile, profile-reference mismatch, or changed SMTP/IMAP transport revision stops delivery. Synchronous Mail batches preflight DNS, connectivity, TLS, and authentication before their first effect, reuse a bounded healthy SMTP connection, and reconnect before a later message when the old connection is stale. Review and send shows batch, connection, reconnect, failure, and pause counts. A systemic authentication, sender, or connectivity failure pauses remaining queued jobs with a stable reason code; correct and test the Mail profile before explicitly resuming. A connection loss after DATA begins stays outcome-unknown and is never replayed automatically. Preserve a stopped record, migrate or correct the profile selection, revalidate, rebuild, and only then queue again. Password-only rotation remains possible without copying secrets into Campaign. Uncertain SMTP and IMAP effects remain blocked until an evidence-backed operator reconciliation. If Campaign becomes unavailable to the tenant after a job was accepted, the worker leaves the job untouched and reports an operator action instead of sending or dropping it.",
layer="configured",
documentation_types=("admin",),
audience=("campaign_sender", "campaign_operator", "mail_admin"),
src/govoplan_campaign/backend/response_security.py
+5
View File
@@ -48,7 +48,12 @@ _SEND_NOW_RESULT_KEYS = (
"failed_count",
"outcome_unknown_count",
"skipped_count",
"paused_count",
"preflight_count",
"batch_state",
"batch_pause_reason_code",
"smtp_connection_count",
"smtp_reconnect_count",
"delivery_mode",
"dry_run",
)
src/govoplan_campaign/backend/sending/jobs.py
+182 -27
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
import json
from collections import Counter
from contextlib import nullcontext
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from email import policy
@@ -225,6 +226,11 @@ class SendCampaignNowResult:
failed_count: int
outcome_unknown_count: int
skipped_count: int
paused_count: int = 0
batch_state: str = "not_started"
batch_pause_reason_code: str | None = None
smtp_connection_count: int = 0
smtp_reconnect_count: int = 0
preflight_count: int = 0
synchronous_send_policy: dict[str, Any] | None = None
dry_run: bool = False
@@ -239,7 +245,12 @@ class SendCampaignNowResult:
"failed_count": self.failed_count,
"outcome_unknown_count": self.outcome_unknown_count,
"skipped_count": self.skipped_count,
"paused_count": self.paused_count,
"preflight_count": self.preflight_count,
"batch_state": self.batch_state,
"batch_pause_reason_code": self.batch_pause_reason_code,
"smtp_connection_count": self.smtp_connection_count,
"smtp_reconnect_count": self.smtp_reconnect_count,
"delivery_mode": "synchronous",
"synchronous_send_policy": self.synchronous_send_policy or {},
"dry_run": self.dry_run,
@@ -1070,48 +1081,91 @@ def send_campaign_now(
jobs=jobs,
policy=synchronous_policy,
)
# Queue state and its inbox notification become durable only after every
# message and the selected transport revision have passed preflight. This
# preserves late-ack recovery without leaving rejected work eligible for a
# background worker.
session.commit()
results: list[dict[str, Any]] = []
sent_count = 0
failed_count = 0
outcome_unknown_count = 0
skipped_after_queue = 0
for job in jobs:
try:
result = _deliver_job_with_recovery(
session,
job=job,
context=delivery_contexts[job.id],
use_rate_limit=use_rate_limit,
enqueue_imap_task=enqueue_imap_task,
)
result_dict = result.as_dict()
results.append(result_dict)
if result.status in DELIVERY_ACCEPTED_STATUSES | {"already_accepted"}:
sent_count += 1
elif result.status == JobSendStatus.OUTCOME_UNKNOWN.value:
outcome_unknown_count += 1
else:
skipped_after_queue += 1
except Exception as exc: # keep sending other jobs and return per-job details
failed_count += 1
results.append({"job_id": job.id, "status": "failed", "message": str(exc)})
attempted_count = 0
paused_count = 0
pause_reason_code: str | None = None
batch_state = "ready"
batch_manager = _synchronous_smtp_batch_manager(
session,
jobs=jobs,
contexts=delivery_contexts,
)
try:
with batch_manager as smtp_batch:
# Queue state becomes durable only after local and SMTP
# DNS/connectivity/TLS/auth preflight succeeds.
session.commit()
for index, job in enumerate(jobs):
attempted_count += 1
try:
result = _deliver_job_with_recovery(
session,
job=job,
context=delivery_contexts[job.id],
use_rate_limit=use_rate_limit,
enqueue_imap_task=enqueue_imap_task,
)
result_dict = result.as_dict()
results.append(result_dict)
if result.status in DELIVERY_ACCEPTED_STATUSES | {"already_accepted"}:
sent_count += 1
elif result.status == JobSendStatus.OUTCOME_UNKNOWN.value:
outcome_unknown_count += 1
else:
skipped_after_queue += 1
except Exception as exc:
failed_count += 1
results.append({"job_id": job.id, "status": "failed", "message": str(exc)})
if isinstance(exc, SmtpSendError) and exc.systemic:
pause_reason_code = exc.reason_code or "smtp_systemic_failure"
paused_count = _pause_jobs_after_systemic_smtp_failure(
session,
campaign_id=job.campaign_id,
exclude_job_id=job.id,
reason_code=pause_reason_code,
)
batch_state = "paused"
for remaining in jobs[index + 1 :]:
results.append(
{
"job_id": remaining.id,
"status": "paused",
"message": "Batch paused after a systemic SMTP failure.",
}
)
break
smtp_connection_count = int(getattr(smtp_batch, "connection_count", 0) or 0)
smtp_reconnect_count = int(getattr(smtp_batch, "reconnect_count", 0) or 0)
except (MailProfileError, SmtpConfigurationError, SmtpSendError, OSError) as exc:
session.rollback()
reason_code = str(getattr(exc, "reason_code", "") or "smtp_batch_preflight_failed")
raise SynchronousSendRejected(
"SMTP batch preflight could not validate DNS, connectivity, TLS, and authentication; no message was sent.",
reason=reason_code,
eligible_count=len(jobs),
policy=synchronous_policy,
) from exc
return SendCampaignNowResult(
campaign_id=campaign.id,
version_id=version.id,
attempted_count=len(jobs),
attempted_count=attempted_count,
sent_count=sent_count,
failed_count=failed_count,
outcome_unknown_count=outcome_unknown_count,
skipped_count=queue_result.skipped_count
+ queue_result.blocked_count
+ skipped_after_queue,
paused_count=paused_count,
batch_state=batch_state,
batch_pause_reason_code=pause_reason_code,
smtp_connection_count=smtp_connection_count,
smtp_reconnect_count=smtp_reconnect_count,
preflight_count=len(delivery_contexts),
synchronous_send_policy=synchronous_policy.as_dict(),
dry_run=False,
@@ -1193,6 +1247,100 @@ def _preflight_synchronous_send_batch(
return contexts
def _synchronous_smtp_batch_manager(
session: Session,
*,
jobs: list[CampaignJob],
contexts: dict[str, _SendJobDeliveryContext],
):
mail_items = [
(job, contexts[job.id])
for job in jobs
if DeliveryChannelPolicy(job.delivery_channel_policy).uses_mail
]
if not mail_items:
return nullcontext(None)
first_job, first_context = mail_items[0]
envelope_froms = {str(context.envelope_from or "") for _job, context in mail_items}
transport_keys = {
(
context.snapshot.mail_profile_id,
context.snapshot.smtp_transport_revision,
context.snapshot.smtp_server_id,
context.snapshot.smtp_credential_id,
)
for _job, context in mail_items
}
if len(envelope_froms) != 1 or "" in envelope_froms or len(transport_keys) != 1:
raise SynchronousSendRejected(
"A synchronous SMTP batch requires one frozen sender and transport selection.",
reason="smtp_batch_transport_mismatch",
eligible_count=len(jobs),
)
recipients = sorted(
{
recipient
for _job, context in mail_items
for recipient in context.envelope_recipients
}
)
return mail_integration().campaign_smtp_batch(
session,
tenant_id=first_job.tenant_id,
campaign_id=first_job.campaign_id,
profile_id=first_context.snapshot.mail_profile_id,
envelope_from=str(first_context.envelope_from),
envelope_recipients=recipients,
from_header=_from_header_from_job(first_job),
expected_smtp_transport_revision=first_context.snapshot.smtp_transport_revision or "",
smtp_server_id=first_context.snapshot.smtp_server_id,
smtp_credential_id=first_context.snapshot.smtp_credential_id,
)
def _pause_jobs_after_systemic_smtp_failure(
session: Session,
*,
campaign_id: str,
exclude_job_id: str,
reason_code: str,
) -> int:
reason = f"SMTP batch paused after systemic failure ({reason_code[:80]})."
changed = (
session.query(CampaignJob)
.filter(
CampaignJob.campaign_id == campaign_id,
CampaignJob.id != exclude_job_id,
CampaignJob.queue_status == JobQueueStatus.QUEUED.value,
CampaignJob.send_status.in_(
[JobSendStatus.QUEUED.value, JobSendStatus.FAILED_TEMPORARY.value]
),
)
.update(
{
CampaignJob.queue_status: JobQueueStatus.PAUSED.value,
CampaignJob.last_error: reason,
},
synchronize_session=False,
)
)
campaign = session.get(Campaign, campaign_id)
if changed and campaign is not None:
campaign.status = CampaignStatus.READY_TO_QUEUE.value
session.add(campaign)
audit_event(
session,
tenant_id=campaign.tenant_id if campaign is not None else None,
user_id=None,
action="campaign.smtp_batch_paused",
object_type="campaign",
object_id=campaign_id,
details={"reason_code": reason_code[:80], "paused_count": int(changed)},
)
session.commit()
return int(changed)
def enqueue_existing_queued_jobs(
session: Session, *, tenant_id: str, campaign_id: str
) -> int:
@@ -3652,6 +3800,13 @@ def _send_claimed_mail_only_job(
outcome_unknown = _record_smtp_send_error(
session, job=job, attempt=attempt, exc=exc
)
if exc.systemic:
_pause_jobs_after_systemic_smtp_failure(
session,
campaign_id=job.campaign_id,
exclude_job_id=job.id,
reason_code=exc.reason_code or "smtp_systemic_failure",
)
if outcome_unknown is not None:
return outcome_unknown
raise