GovOPlaN/govoplan-campaign
1
0
Fork 0
Code Issues 46 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add governed postbox delivery and report hardening

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-29 14:16:28 +02:00
parent f11c56e890
commit 5240749ae1
47 changed files with 5538 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

499
src/govoplan_campaign/backend/reports/campaigns.py
View File

@@ -5,6 +5,7 @@ import io
import logging
import math
from collections import Counter
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
@@ -18,6 +19,7 @@ from govoplan_campaign.backend.db.models import (
CampaignJob,
CampaignVersion,
ImapAppendAttempt,
PostboxDeliveryAttempt,
SendAttempt,
)
from govoplan_campaign.backend.sending.execution import ExecutionSnapshot
@@ -33,6 +35,8 @@ class CampaignReportError(RuntimeError):
logger = logging.getLogger(__name__)
CAMPAIGN_JSON_JOB_LIMIT = 5_000
CAMPAIGN_CSV_JOB_LIMIT = 100_000
def _utcnow_iso() -> str:
@@ -96,8 +100,9 @@ def _version_info(
def _load_delivery_info(
version: CampaignVersion | None,
jobs: list[CampaignJob],
jobs: list[CampaignJob] | None = None,
*,
pending_job_count: int | None = None,
include_diagnostics: bool = False,
) -> dict[str, Any]:
"""Read deterministic delivery settings from the immutable execution snapshot."""
@@ -143,10 +148,17 @@ def _load_delivery_info(
return default
messages_per_minute = snapshot.delivery.rate_limit.messages_per_minute
pending = [job for job in jobs if job.send_status in {"queued", "claimed", "sending"}]
if pending_job_count is None:
pending_job_count = sum(
1
for job in jobs or []
if job.send_status in {"queued", "claimed", "sending"}
)
estimated_seconds = None
if messages_per_minute and pending:
estimated_seconds = int(math.ceil((len(pending) / messages_per_minute) * 60))
if messages_per_minute and pending_job_count:
estimated_seconds = int(
math.ceil((pending_job_count / messages_per_minute) * 60)
)
result = {
"rate_limit": {
@@ -259,8 +271,167 @@ def _attachment_summary(jobs: list[CampaignJob]) -> dict[str, Any]:
}
@dataclass(slots=True)
class _JobReportAggregate:
total: int = 0
pending: int = 0
queueable: int = 0
queueable_unattempted: int = 0
retryable: int = 0
cancellable: int = 0
needs_attention: int = 0
build: Counter[str] = field(default_factory=Counter)
validation: Counter[str] = field(default_factory=Counter)
queue: Counter[str] = field(default_factory=Counter)
send: Counter[str] = field(default_factory=Counter)
postbox: Counter[str] = field(default_factory=Counter)
imap: Counter[str] = field(default_factory=Counter)
issue_total: int = 0
issue_severity: Counter[str] = field(default_factory=Counter)
issue_code: Counter[str] = field(default_factory=Counter)
issue_behavior: Counter[str] = field(default_factory=Counter)
attachment_total: int = 0
attachment_matches: int = 0
attachment_zip_enabled: int = 0
attachment_missing: int = 0
attachment_ambiguous: int = 0
attachment_status: Counter[str] = field(default_factory=Counter)
attachment_behavior: Counter[str] = field(default_factory=Counter)
recent_failures: list[CampaignJob] = field(default_factory=list)
def add(
self,
job: CampaignJob,
*,
retry_max_attempts: int | None,
include_recent_failures: bool,
) -> None:
self.total += 1
self.build[job.build_status or "unknown"] += 1
self.validation[job.validation_status or "unknown"] += 1
self.queue[job.queue_status or "unknown"] += 1
self.send[job.send_status or "unknown"] += 1
self.postbox[job.postbox_status or "unknown"] += 1
self.imap[job.imap_status or "unknown"] += 1
if job.send_status in {"queued", "claimed", "sending"}:
self.pending += 1
if (
job.validation_status in {"ready", "warning"}
and job.build_status == "built"
):
self.queueable += 1
if (
job.attempt_count == 0
and job.postbox_attempt_count == 0
and job.send_status in {"not_queued", "cancelled"}
and job.validation_status in {"ready", "warning"}
and job.build_status == "built"
):
self.queueable_unattempted += 1
if (
job.send_status in {"failed_temporary", "partially_accepted"}
and (
retry_max_attempts is None
or job.attempt_count < retry_max_attempts
)
):
self.retryable += 1
if job.send_status not in {
"skipped",
"smtp_accepted",
"postbox_accepted",
"delivered",
"partially_accepted",
"sent",
"outcome_unknown",
"claimed",
"sending",
"cancelled",
}:
self.cancellable += 1
if _job_needs_attention(job):
self.needs_attention += 1
self._add_issues(job)
self._add_attachments(job)
if include_recent_failures and _job_is_recent_failure(job):
self.recent_failures.append(job)
self.recent_failures.sort(
key=lambda item: item.updated_at or item.created_at,
reverse=True,
)
del self.recent_failures[20:]
def _add_issues(self, job: CampaignJob) -> None:
for issue in job.issues_snapshot or []:
if not isinstance(issue, dict):
continue
self.issue_total += 1
self.issue_severity[issue.get("severity") or "unknown"] += 1
self.issue_code[issue.get("code") or "unknown"] += 1
if issue.get("behavior"):
self.issue_behavior[str(issue["behavior"])] += 1
def _add_attachments(self, job: CampaignJob) -> None:
for attachment in job.resolved_attachments or []:
if not isinstance(attachment, dict):
continue
self.attachment_total += 1
status_value = str(attachment.get("status") or "unknown")
self.attachment_status[status_value] += 1
if attachment.get("behavior"):
self.attachment_behavior[str(attachment["behavior"])] += 1
matches = attachment.get("matches")
if isinstance(matches, list):
self.attachment_matches += len(matches)
if attachment.get("zip_enabled"):
self.attachment_zip_enabled += 1
if status_value == "missing":
self.attachment_missing += 1
if status_value == "ambiguous":
self.attachment_ambiguous += 1
def _job_needs_attention(job: CampaignJob) -> bool:
return (
job.validation_status in {"needs_review", "blocked"}
or job.send_status
in {
"failed_temporary",
"failed_permanent",
"partially_accepted",
"outcome_unknown",
"claimed",
"sending",
}
or job.postbox_status
in {
"partially_accepted",
"rejected_temporary",
"rejected_permanent",
"outcome_unknown",
}
or job.imap_status == "failed"
)
def _job_is_recent_failure(job: CampaignJob) -> bool:
return bool(
job.last_error
or str(job.send_status).startswith("failed")
or job.send_status == "partially_accepted"
or job.postbox_status
in {
"partially_accepted",
"rejected_temporary",
"rejected_permanent",
"outcome_unknown",
}
or job.imap_status == "failed"
)
def _recent_failures(jobs: list[CampaignJob], *, limit: int = 20) -> list[dict[str, Any]]:
failed = [job for job in jobs if job.last_error or str(job.send_status).startswith("failed") or job.imap_status == "failed"]
failed = [job for job in jobs if _job_is_recent_failure(job)]
failed.sort(key=lambda job: job.updated_at or job.created_at, reverse=True)
return [
{
@@ -270,12 +441,15 @@ def _recent_failures(jobs: list[CampaignJob], *, limit: int = 20) -> list[dict[s
"recipient_email": job.recipient_email,
"validation_status": job.validation_status,
"send_status": job.send_status,
"postbox_status": job.postbox_status,
"imap_status": job.imap_status,
"attempt_count": job.attempt_count,
"postbox_attempt_count": job.postbox_attempt_count,
"last_error": public_delivery_result_message(
last_error=job.last_error,
send_status=job.send_status,
imap_status=job.imap_status,
postbox_status=job.postbox_status,
),
"updated_at": job.updated_at.isoformat() if job.updated_at else None,
}
@@ -294,8 +468,22 @@ def _job_row(job: CampaignJob, *, include_diagnostics: bool = False) -> dict[str
"validation_status": job.validation_status,
"queue_status": job.queue_status,
"send_status": job.send_status,
"delivery_channel_policy": getattr(
job,
"delivery_channel_policy",
"mail",
),
"postbox_status": getattr(
job,
"postbox_status",
"not_requested",
),
"imap_status": job.imap_status,
"attempt_count": job.attempt_count,
"postbox_attempt_count": getattr(job, "postbox_attempt_count", 0),
"postbox_target_count": len(
getattr(job, "resolved_postbox_targets", None) or []
),
"queued_at": job.queued_at.isoformat() if job.queued_at else None,
"outcome_unknown_at": job.outcome_unknown_at.isoformat() if job.outcome_unknown_at else None,
"sent_at": job.sent_at.isoformat() if job.sent_at else None,
@@ -303,6 +491,7 @@ def _job_row(job: CampaignJob, *, include_diagnostics: bool = False) -> dict[str
last_error=job.last_error,
send_status=job.send_status,
imap_status=job.imap_status,
postbox_status=getattr(job, "postbox_status", "not_requested"),
),
"eml_size_bytes": job.eml_size_bytes,
"eml_sha256": job.eml_sha256,
@@ -393,6 +582,13 @@ def _job_evidence_row(
"cc": _address_summary(recipients.get("cc")),
"bcc": _address_summary(recipients.get("bcc")),
"reply_to": _address_summary(recipients.get("reply_to")),
"postbox_targets": "; ".join(
str(target.get("address") or target.get("name") or target.get("postbox_id") or "")
for target in (
getattr(job, "resolved_postbox_targets", None) or []
)
if isinstance(target, dict)
),
"attachment_names": _attachment_names(job.resolved_attachments),
"latest_smtp_attempt_number": latest_smtp.attempt_number if latest_smtp else None,
"latest_smtp_status": latest_smtp.status if latest_smtp else None,
@@ -427,21 +623,43 @@ def generate_campaign_report(
campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
version = _selected_version(session, campaign, version_id)
jobs = _report_jobs(session, tenant_id=tenant_id, campaign_id=campaign.id, version=version)
jobs = _report_jobs(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version=version,
)
aggregate = _JobReportAggregate()
job_rows: list[dict[str, Any]] = []
retry_max_attempts = _retry_max_attempts(version)
iterator = jobs.yield_per(500) if hasattr(jobs, "yield_per") else jobs
for job in iterator:
aggregate.add(
job,
retry_max_attempts=retry_max_attempts,
include_recent_failures=include_recent_failures,
)
if include_jobs:
if len(job_rows) >= CAMPAIGN_JSON_JOB_LIMIT:
raise CampaignReportError(
"The recipient-level JSON report exceeds the safe row "
f"limit of {CAMPAIGN_JSON_JOB_LIMIT}; use the CSV export "
"or the paginated recipient table."
)
job_rows.append(
_job_row(job, include_diagnostics=include_diagnostics)
)
report = _campaign_report_payload(
session,
campaign=campaign,
version=version,
jobs=jobs,
aggregate=aggregate,
tenant_id=tenant_id,
include_recent_failures=include_recent_failures,
include_diagnostics=include_diagnostics,
)
if include_jobs:
report["jobs"] = [
_job_row(job, include_diagnostics=include_diagnostics)
for job in jobs
]
report["jobs"] = job_rows
return report
@@ -451,7 +669,7 @@ def _report_jobs(
tenant_id: str,
campaign_id: str,
version: CampaignVersion | None,
) -> list[CampaignJob]:
) -> Any:
jobs_query = session.query(CampaignJob).filter(
CampaignJob.tenant_id == tenant_id,
CampaignJob.campaign_id == campaign_id,
@@ -460,7 +678,7 @@ def _report_jobs(
jobs_query = jobs_query.filter(CampaignJob.campaign_version_id == version.id)
else:
jobs_query = jobs_query.filter(False)
return jobs_query.order_by(CampaignJob.entry_index.asc()).all()
return jobs_query.order_by(CampaignJob.entry_index.asc())
def _campaign_report_payload(
@@ -468,21 +686,26 @@ def _campaign_report_payload(
*,
campaign: Campaign,
version: CampaignVersion | None,
jobs: list[CampaignJob],
aggregate: _JobReportAggregate,
tenant_id: str,
include_recent_failures: bool,
include_diagnostics: bool,
) -> dict[str, Any]:
job_ids = [job.id for job in jobs]
report = {
"generated_at": _utcnow_iso(),
"campaign": _campaign_report_campaign_payload(campaign),
"current_version": _version_info(version, include_diagnostics=include_diagnostics),
"selected_version_id": version.id if version else None,
"cards": _campaign_report_cards(version, jobs),
"status_counts": _campaign_report_status_counts(version, jobs),
"cards": _campaign_report_cards_from_aggregate(version, aggregate),
"status_counts": _campaign_report_status_counts_from_aggregate(
version,
aggregate,
),
"issues": {
**_issue_summary_from_jobs(jobs),
"total": aggregate.issue_total,
"by_severity": dict(aggregate.issue_severity),
"by_code": dict(aggregate.issue_code),
"by_behavior": dict(aggregate.issue_behavior),
"persisted_campaign_issue_count": _persisted_campaign_issue_count(
session,
tenant_id=tenant_id,
@@ -490,16 +713,31 @@ def _campaign_report_payload(
version=version,
),
},
"attachments": _attachment_summary(jobs),
"attempts": _campaign_report_attempt_counts(session, job_ids),
"attachments": {
"total_attachment_configs": aggregate.attachment_total,
"total_matched_files": aggregate.attachment_matches,
"zip_enabled_configs": aggregate.attachment_zip_enabled,
"missing_configs": aggregate.attachment_missing,
"ambiguous_configs": aggregate.attachment_ambiguous,
"by_status": dict(aggregate.attachment_status),
"by_behavior": dict(aggregate.attachment_behavior),
},
"attempts": _campaign_report_attempt_counts(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version=version,
),
"delivery": _load_delivery_info(
version,
jobs,
pending_job_count=aggregate.pending,
include_diagnostics=include_diagnostics,
),
}
if include_recent_failures:
report["recent_failures"] = _recent_failures(jobs)
report["recent_failures"] = _recent_failures(
aggregate.recent_failures,
)
return report
@@ -515,10 +753,34 @@ def _campaign_report_campaign_payload(campaign: Campaign) -> dict[str, Any]:
}
def _campaign_report_attempt_counts(session: Session, job_ids: list[str]) -> dict[str, int]:
def _campaign_report_attempt_counts(
session: Session,
*,
tenant_id: str,
campaign_id: str,
version: CampaignVersion | None,
) -> dict[str, int]:
def count(model: type[Any]) -> int:
query = (
session.query(model)
.join(CampaignJob, model.job_id == CampaignJob.id)
.filter(
CampaignJob.tenant_id == tenant_id,
CampaignJob.campaign_id == campaign_id,
)
)
if version is not None:
query = query.filter(
CampaignJob.campaign_version_id == version.id,
)
else:
query = query.filter(False)
return int(query.count())
return {
"send_attempts": int(session.query(SendAttempt).filter(SendAttempt.job_id.in_(job_ids)).count()) if job_ids else 0,
"imap_append_attempts": int(session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id.in_(job_ids)).count()) if job_ids else 0,
"send_attempts": count(SendAttempt),
"imap_append_attempts": count(ImapAppendAttempt),
"postbox_delivery_attempts": count(PostboxDeliveryAttempt),
}
@@ -541,84 +803,109 @@ def _persisted_campaign_issue_count(
def _campaign_report_status_counts(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, dict[str, int]]:
validation_counts = _counter([job.validation_status for job in jobs])
aggregate = _aggregate_job_list(version, jobs)
return _campaign_report_status_counts_from_aggregate(version, aggregate)
def _campaign_report_status_counts_from_aggregate(
version: CampaignVersion | None,
aggregate: _JobReportAggregate,
) -> dict[str, dict[str, int]]:
validation_counts = dict(aggregate.validation)
inactive_entries = _inactive_entry_count(version)
if inactive_entries:
validation_counts["inactive"] = inactive_entries
return {
"build": _counter([job.build_status for job in jobs]),
"build": dict(aggregate.build),
"validation": validation_counts,
"queue": _counter([job.queue_status for job in jobs]),
"send": _counter([job.send_status for job in jobs]),
"imap": _counter([job.imap_status for job in jobs]),
"queue": dict(aggregate.queue),
"send": dict(aggregate.send),
"postbox": dict(aggregate.postbox),
"imap": dict(aggregate.imap),
}
def _campaign_report_cards(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, object]:
send_counts = _counter([job.send_status for job in jobs])
imap_counts = _counter([job.imap_status for job in jobs])
queueable = sum(1 for job in jobs if job.validation_status in {"ready", "warning"} and job.build_status == "built")
queueable_unattempted = sum(
1
for job in jobs
if job.attempt_count == 0
and job.send_status in {"not_queued", "cancelled"}
and job.validation_status in {"ready", "warning"}
and job.build_status == "built"
def _campaign_report_cards_from_aggregate(
version: CampaignVersion | None,
aggregate: _JobReportAggregate,
) -> dict[str, object]:
send_counts = aggregate.send
imap_counts = aggregate.imap
sent = (
send_counts.get("sent", 0)
+ send_counts.get("smtp_accepted", 0)
+ send_counts.get("postbox_accepted", 0)
+ send_counts.get("delivered", 0)
+ send_counts.get("partially_accepted", 0)
)
retry_max_attempts = _retry_max_attempts(version)
retryable = sum(
1
for job in jobs
if job.send_status == "failed_temporary"
and (retry_max_attempts is None or job.attempt_count < retry_max_attempts)
failed = (
send_counts.get("failed_temporary", 0)
+ send_counts.get("failed_permanent", 0)
)
cancellable = sum(
1
for job in jobs
if job.send_status
not in {"skipped", "smtp_accepted", "sent", "outcome_unknown", "claimed", "sending", "cancelled"}
)
needs_attention = sum(
1
for job in jobs
if job.validation_status in {"needs_review", "blocked"}
or job.send_status in {"failed_temporary", "failed_permanent", "outcome_unknown", "claimed", "sending"}
or job.imap_status == "failed"
)
sent = send_counts.get("sent", 0) + send_counts.get("smtp_accepted", 0)
failed = send_counts.get("failed_temporary", 0) + send_counts.get("failed_permanent", 0)
outcome_unknown = send_counts.get("outcome_unknown", 0)
not_attempted = send_counts.get("not_queued", 0)
skipped = send_counts.get("skipped", 0)
queued = send_counts.get("queued", 0) + send_counts.get("claimed", 0) + send_counts.get("sending", 0)
cancelled = send_counts.get("cancelled", 0)
inactive_entries = _inactive_entry_count(version)
return {
"jobs_total": len(jobs),
"jobs_total": aggregate.total,
"inactive": inactive_entries,
"queueable": queueable,
"queueable_unattempted": queueable_unattempted,
"retryable": retryable,
"cancellable": cancellable,
"needs_attention": needs_attention,
"queueable": aggregate.queueable,
"queueable_unattempted": aggregate.queueable_unattempted,
"retryable": aggregate.retryable,
"cancellable": aggregate.cancellable,
"needs_attention": aggregate.needs_attention,
"sent": sent,
"smtp_accepted": sent,
"smtp_accepted": (
send_counts.get("sent", 0)
+ send_counts.get("smtp_accepted", 0)
),
"postbox_accepted": send_counts.get("postbox_accepted", 0),
"delivered": send_counts.get("delivered", 0),
"partially_accepted": send_counts.get("partially_accepted", 0),
"failed": failed,
"outcome_unknown": outcome_unknown,
"not_attempted": not_attempted,
"skipped": skipped,
"queued_or_active": queued,
"skipped": send_counts.get("skipped", 0),
"queued_or_active": aggregate.pending,
"cancelled": cancelled,
"partially_completed": bool(sent and (failed or outcome_unknown or not_attempted or cancelled)),
"partially_completed": bool(
send_counts.get("partially_accepted", 0)
or sent
and (failed or outcome_unknown or not_attempted or cancelled)
),
"imap_appended": imap_counts.get("appended", 0),
"imap_failed": imap_counts.get("failed", 0),
"imap_skipped": imap_counts.get("skipped", 0),
}
def _campaign_report_cards(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, object]:
return _campaign_report_cards_from_aggregate(
version,
_aggregate_job_list(version, jobs),
)
def _aggregate_job_list(
version: CampaignVersion | None,
jobs: list[CampaignJob],
) -> _JobReportAggregate:
aggregate = _JobReportAggregate()
retry_max_attempts = _retry_max_attempts(version)
for job in jobs:
aggregate.add(
job,
retry_max_attempts=retry_max_attempts,
include_recent_failures=False,
)
return aggregate
def _retry_max_attempts(version: CampaignVersion | None) -> int | None:
if version is None or not isinstance(version.execution_snapshot, dict):
if version is None or not isinstance(
getattr(version, "execution_snapshot", None),
dict,
):
return None
try:
return ExecutionSnapshot.model_validate(version.execution_snapshot).delivery.retry.max_attempts
@@ -652,27 +939,25 @@ def generate_jobs_csv(
jobs = (
jobs_query
.order_by(CampaignJob.entry_index.asc())
.limit(CAMPAIGN_CSV_JOB_LIMIT + 1)
.all()
)
if len(jobs) > CAMPAIGN_CSV_JOB_LIMIT:
raise CampaignReportError(
"The campaign CSV export exceeds the safe row limit of "
f"{CAMPAIGN_CSV_JOB_LIMIT}. Split the export by recipient filter."
)
job_ids = [job.id for job in jobs]
smtp_attempts = (
session.query(SendAttempt)
.filter(SendAttempt.job_id.in_(job_ids))
.order_by(SendAttempt.job_id.asc(), SendAttempt.attempt_number.asc())
.all()
if job_ids
else []
latest_smtp = _latest_attempts_for_jobs(
session,
SendAttempt,
job_ids,
)
imap_attempts = (
session.query(ImapAppendAttempt)
.filter(ImapAppendAttempt.job_id.in_(job_ids))
.order_by(ImapAppendAttempt.job_id.asc(), ImapAppendAttempt.attempt_number.asc())
.all()
if job_ids
else []
latest_imap = _latest_attempts_for_jobs(
session,
ImapAppendAttempt,
job_ids,
)
latest_smtp = _latest_by_job_id(smtp_attempts)
latest_imap = _latest_by_job_id(imap_attempts)
rows = [
_job_evidence_row(
job,
@@ -700,8 +985,13 @@ def generate_jobs_csv(
"validation_status",
"queue_status",
"send_status",
"delivery_channel_policy",
"postbox_status",
"imap_status",
"attempt_count",
"postbox_attempt_count",
"postbox_target_count",
"postbox_targets",
"queued_at",
"outcome_unknown_at",
"sent_at",
@@ -731,3 +1021,30 @@ def generate_jobs_csv(
writer.writeheader()
writer.writerows(rows)
return buffer.getvalue()
def _latest_attempts_for_jobs(
session: Session,
model: type[Any],
job_ids: list[str],
*,
batch_size: int = 500,
) -> dict[str, Any]:
latest: dict[str, Any] = {}
for offset in range(0, len(job_ids), batch_size):
batch = job_ids[offset : offset + batch_size]
attempts = (
session.query(model)
.filter(model.job_id.in_(batch))
.order_by(model.job_id.asc(), model.attempt_number.asc())
.yield_per(batch_size)
)
for attempt in attempts:
current = latest.get(attempt.job_id)
if (
current is None
or (attempt.attempt_number or 0)
>= (current.attempt_number or 0)
):
latest[attempt.job_id] = attempt
return latest