GovOPlaN/govoplan-campaign
1
0
Fork 0
Code Issues 46 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add governed postbox delivery and report hardening

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-29 14:16:28 +02:00
parent f11c56e890
commit 5240749ae1
47 changed files with 5538 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
src/govoplan_campaign/backend/router.py
View File

@@ -4,7 +4,7 @@ import copy
import dataclasses
import json
import logging
from collections.abc import Callable
from collections.abc import Callable, Sequence
from typing import Any, Literal
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
@@ -29,6 +29,7 @@ from govoplan_campaign.backend.schemas import (
CampaignOwnerUpdateRequest,
CampaignAddressLookupCandidate,
CampaignAddressLookupResponse,
CampaignPostboxCatalogResponse,
CampaignRecipientAddressSource,
CampaignRecipientAddressSourcesResponse,
CampaignRecipientAddressSourceSnapshotRequest,
@@ -98,12 +99,21 @@ from govoplan_campaign.backend.db.models import (
CampaignVersionWorkflowState,
ImapAppendAttempt,
JobImapStatus,
JobPostboxStatus,
JobQueueStatus,
JobSendStatus,
JobValidationStatus,
RecipientImportMappingProfile,
PostboxDeliveryAttempt,
SendAttempt,
)
from govoplan_campaign.backend.campaign.postbox_targets import (
delivery_catalog_payload,
)
from govoplan_campaign.backend.integrations import (
PostboxDeliveryUnavailable,
postbox_integration,
)
from govoplan_core.db.session import get_session
from govoplan_campaign.backend.reports.campaigns import CampaignReportError, generate_campaign_report, generate_jobs_csv
from govoplan_campaign.backend.report_privacy_policy import CampaignReportPrivacyPolicyError
@@ -719,14 +729,38 @@ def create_minimal_campaign_endpoint(
@router.get("", response_model=CampaignListResponse)
def list_campaigns(
limit: int = 200,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:campaign:read")),
):
campaigns = _campaign_query_for_principal(session, principal).order_by(Campaign.updated_at.desc()).all()
campaigns = (
_campaign_query_for_principal(session, principal)
.order_by(Campaign.updated_at.desc())
.limit(max(1, min(limit, 500)))
.all()
)
return CampaignListResponse(campaigns=[CampaignResponse.model_validate(item) for item in campaigns])
def _bounded_query_rows(query, *, limit: int, label: str):
rows = query.limit(limit + 1).all()
if len(rows) > limit:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail=(
f"{label} exceeds the maximum response size of {limit} rows. "
"Narrow the request or use a paginated/delta endpoint."
),
)
return rows
def _job_attempt_rows(query, *, label: str):
return _bounded_query_rows(query, limit=1000, label=label)
_CAMPAIGN_LIST_DELTA_COLLECTIONS = (CAMPAIGNS_COLLECTION,)
_CAMPAIGN_FULL_CURSOR_PREFIX = "full:campaigns:"
_CAMPAIGN_WORKSPACE_DELTA_COLLECTIONS = (
CAMPAIGNS_COLLECTION,
CAMPAIGN_VERSIONS_COLLECTION,
@@ -786,14 +820,89 @@ def _campaign_entry_matches_principal(session: Session, principal: ApiPrincipal,
return False
def _campaign_list_full_delta_response(session: Session, principal: ApiPrincipal) -> CampaignDeltaResponse:
campaigns = _campaign_query_for_principal(session, principal).order_by(Campaign.updated_at.desc()).all()
def _campaign_full_cursor(
*,
page: int,
snapshot_sequence: int,
) -> str:
return (
f"{_CAMPAIGN_FULL_CURSOR_PREFIX}"
f"{int(page)}:{int(snapshot_sequence)}"
)
def _decode_campaign_full_cursor(
value: str | None,
) -> tuple[int, int] | None:
if not value or not value.startswith(_CAMPAIGN_FULL_CURSOR_PREFIX):
return None
parts = value[len(_CAMPAIGN_FULL_CURSOR_PREFIX):].split(":", 1)
if len(parts) != 2:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid campaign full snapshot cursor",
)
try:
page, snapshot_sequence = (int(item) for item in parts)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid campaign full snapshot cursor",
) from exc
if page < 1 or snapshot_sequence < 0:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Invalid campaign full snapshot cursor",
)
return page, snapshot_sequence
def _campaign_list_full_delta_response(
session: Session,
principal: ApiPrincipal,
*,
cursor: tuple[int, int] | None = None,
limit: int = 500,
) -> CampaignDeltaResponse:
page = cursor[0] if cursor is not None else 1
snapshot_sequence = (
cursor[1]
if cursor is not None
else decode_sequence_watermark(
_campaign_delta_watermark(
session,
principal.tenant_id,
_CAMPAIGN_LIST_DELTA_COLLECTIONS,
),
)
)
query = _campaign_query_for_principal(session, principal)
total = query.order_by(None).count()
pages = max(1, (total + limit - 1) // limit)
campaigns = (
query.order_by(Campaign.updated_at.desc(), Campaign.id.asc())
.offset((page - 1) * limit)
.limit(limit)
.all()
)
has_more = page < pages
return CampaignDeltaResponse(
campaigns=[CampaignResponse.model_validate(item) for item in campaigns],
deleted=[],
watermark=_campaign_delta_watermark(session, principal.tenant_id, _CAMPAIGN_LIST_DELTA_COLLECTIONS),
has_more=False,
watermark=(
_campaign_full_cursor(
page=page + 1,
snapshot_sequence=snapshot_sequence,
)
if has_more
else encode_sequence_watermark(snapshot_sequence)
),
has_more=has_more,
full=True,
total=total,
page=page,
page_size=limit,
pages=pages,
)
@@ -809,7 +918,11 @@ def _campaign_list_delta_response(session: Session, principal: ApiPrincipal, *,
module_id=CAMPAIGNS_MODULE_ID,
collections=_CAMPAIGN_LIST_DELTA_COLLECTIONS,
):
return _campaign_list_full_delta_response(session, principal)
return _campaign_list_full_delta_response(
session,
principal,
limit=limit,
)
entries_plus_one = sequence_entries_since(
session,
@@ -858,6 +971,10 @@ def _campaign_list_delta_response(session: Session, principal: ApiPrincipal, *,
watermark=watermark,
has_more=has_more,
full=False,
total=len(visible_campaigns),
page=1,
page_size=limit,
pages=1,
)
@@ -868,24 +985,35 @@ def list_campaigns_delta(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:campaign:read")),
):
if since is None:
return _campaign_list_full_delta_response(session, principal)
full_cursor = _decode_campaign_full_cursor(since)
if since is None or full_cursor is not None:
return _campaign_list_full_delta_response(
session,
principal,
cursor=full_cursor,
limit=limit,
)
return _campaign_list_delta_response(session, principal, since=since, limit=limit)
@router.get("/recipient-import/mapping-profiles", response_model=RecipientImportMappingProfileListResponse)
def list_recipient_import_mapping_profiles(
limit: int = Query(default=200, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:recipient:import")),
):
profiles = (
profiles = _bounded_query_rows(
session.query(RecipientImportMappingProfile)
.filter(
RecipientImportMappingProfile.tenant_id == principal.tenant_id,
RecipientImportMappingProfile.owner_user_id == principal.user.id,
)
.order_by(RecipientImportMappingProfile.updated_at.desc(), RecipientImportMappingProfile.name.asc())
.all()
.order_by(
RecipientImportMappingProfile.updated_at.desc(),
RecipientImportMappingProfile.name.asc(),
),
limit=limit,
label="Recipient import mapping profile list",
)
return RecipientImportMappingProfileListResponse(
profiles=[RecipientImportMappingProfileResponse.model_validate(profile) for profile in profiles]
@@ -978,15 +1106,17 @@ def delete_recipient_import_mapping_profile(
@router.get("/aggregate-reports", response_model=AggregateReportCampaignList)
def list_aggregate_campaign_reports(
limit: int = Query(default=500, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:report:read")),
):
"""List only the business metadata needed to select an aggregate report."""
campaigns = (
campaigns = _bounded_query_rows(
_campaign_query_for_principal(session, principal)
.order_by(Campaign.updated_at.desc())
.all()
.order_by(Campaign.updated_at.desc(), Campaign.id.asc()),
limit=limit,
label="Aggregate report campaign list",
)
return AggregateReportCampaignList(
campaigns=[aggregate_report_campaign_item(campaign) for campaign in campaigns]
@@ -1058,6 +1188,38 @@ def list_campaign_recipient_address_sources(
)
@router.get(
"/{campaign_id}/postbox-catalog",
response_model=CampaignPostboxCatalogResponse,
)
def campaign_postbox_catalog(
campaign_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(
require_scope("campaigns:recipient:write")
),
):
_get_campaign_for_principal(session, campaign_id, principal, write=True)
integration = postbox_integration()
if not integration.available:
return CampaignPostboxCatalogResponse(available=False)
try:
payload = delivery_catalog_payload(
integration.delivery_catalog(
session,
tenant_id=principal.tenant_id,
)
)
except PostboxDeliveryUnavailable:
return CampaignPostboxCatalogResponse(available=False)
return CampaignPostboxCatalogResponse(
available=True,
postboxes=payload.get("postboxes", []),
templates=payload.get("templates", []),
organization_units=payload.get("organization_units", []),
)
@router.post("/{campaign_id}/recipient-address-sources/snapshot", response_model=CampaignRecipientAddressSourceSnapshotResponse)
def snapshot_campaign_recipient_address_source(
campaign_id: str,
@@ -1112,11 +1274,12 @@ def _campaign_workspace_response(
versions: list[CampaignVersion] = []
if include_versions or include_current_version:
versions = (
versions = _bounded_query_rows(
session.query(CampaignVersion)
.filter(CampaignVersion.campaign_id == campaign.id)
.order_by(CampaignVersion.version_number.desc())
.all()
.order_by(CampaignVersion.version_number.desc()),
limit=1000,
label="Campaign version history",
)
selected_version_id = version_id or campaign.current_version_id or (versions[0].id if versions else None)
@@ -1588,16 +1751,18 @@ def delete_draft_campaign(
@router.get("/{campaign_id}/versions", response_model=list[CampaignVersionResponse])
def list_versions(
campaign_id: str,
limit: int = Query(default=500, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:campaign:read")),
):
_get_campaign_for_principal(session, campaign_id, principal)
campaign = _get_campaign_for_tenant(session, campaign_id, principal.tenant_id)
versions = (
versions = _bounded_query_rows(
session.query(CampaignVersion)
.filter(CampaignVersion.campaign_id == campaign.id)
.order_by(CampaignVersion.version_number.desc())
.all()
.order_by(CampaignVersion.version_number.desc()),
limit=limit,
label="Campaign version history",
)
return [
CampaignVersionResponse.model_validate(
@@ -2118,14 +2283,19 @@ def _job_summary_payload(
"validation_status": job.validation_status,
"queue_status": job.queue_status,
"send_status": job.send_status,
"delivery_channel_policy": getattr(job, "delivery_channel_policy", "mail"),
"postbox_status": getattr(job, "postbox_status", "not_requested"),
"imap_status": job.imap_status,
"eml_size_bytes": job.eml_size_bytes,
"eml_sha256": job.eml_sha256,
"attempt_count": job.attempt_count,
"postbox_attempt_count": getattr(job, "postbox_attempt_count", 0),
"postbox_target_count": len(getattr(job, "resolved_postbox_targets", None) or []),
"last_error": public_delivery_result_message(
last_error=job.last_error,
send_status=job.send_status,
imap_status=job.imap_status,
postbox_status=getattr(job, "postbox_status", "not_requested"),
),
"queued_at": job.queued_at,
"outcome_unknown_at": job.outcome_unknown_at,
@@ -2151,12 +2321,14 @@ def _job_detail_payload(job: CampaignJob) -> dict[str, object]:
"issues": job.issues_snapshot or [],
"attachments": public_campaign_payload(job.resolved_attachments or []),
"resolved_recipients": job.resolved_recipients or {},
"resolved_postbox_targets": getattr(job, "resolved_postbox_targets", None) or [],
}
def _job_attempts_payload(
send_attempts: list[SendAttempt],
imap_attempts: list[ImapAppendAttempt],
postbox_attempts: Sequence[PostboxDeliveryAttempt] = (),
*,
include_diagnostics: bool = False,
) -> dict[str, list[dict[str, object]]]:
@@ -2191,13 +2363,43 @@ def _job_attempts_payload(
payload["claim_token"] = attempt.claim_token
payload["error_message"] = attempt.error_message
imap_payloads.append(payload)
return {"smtp": smtp_payloads, "imap": imap_payloads}
postbox_payloads: list[dict[str, object]] = []
for attempt in postbox_attempts:
payload = {
"id": attempt.id,
"target_index": attempt.target_index,
"attempt_number": attempt.attempt_number,
"status": attempt.status,
"postbox_id": attempt.postbox_id,
"address": attempt.address,
"holder_count": attempt.holder_count,
"vacant": attempt.vacant,
"duplicate": attempt.duplicate,
"target": attempt.target_snapshot or {},
"started_at": attempt.started_at,
"finished_at": attempt.finished_at,
"error_code": attempt.error_code,
}
if include_diagnostics:
payload["idempotency_key"] = attempt.idempotency_key
payload["provider_delivery_id"] = attempt.provider_delivery_id
payload["provider_message_id"] = attempt.provider_message_id
payload["evidence"] = attempt.evidence or {}
payload["error_type"] = attempt.error_type
payload["error_message"] = attempt.error_message
postbox_payloads.append(payload)
return {
"smtp": smtp_payloads,
"imap": imap_payloads,
"postbox": postbox_payloads,
}
def _job_diagnostics_payload(
job: CampaignJob,
send_attempts: list[SendAttempt],
imap_attempts: list[ImapAppendAttempt],
postbox_attempts: Sequence[PostboxDeliveryAttempt] = (),
) -> CampaignJobDiagnosticsResponse:
return CampaignJobDiagnosticsResponse(
job_id=job.id,
@@ -2219,6 +2421,7 @@ def _job_diagnostics_payload(
attempts=_job_attempts_payload(
send_attempts,
imap_attempts,
postbox_attempts,
include_diagnostics=True,
),
)
@@ -2307,7 +2510,14 @@ def _review_metadata_counts(review_rows: list[tuple[object, int, str, str]], rev
def _status_counts(session: Session, filters: list[object]) -> dict[str, dict[str, int]]:
result: dict[str, dict[str, int]] = {}
for field_name in ("build_status", "validation_status", "queue_status", "send_status", "imap_status"):
for field_name in (
"build_status",
"validation_status",
"queue_status",
"send_status",
"postbox_status",
"imap_status",
):
column = getattr(CampaignJob, field_name)
rows = session.query(column, func.count(CampaignJob.id)).filter(*filters).group_by(column).all()
result[field_name.removesuffix("_status")] = {str(value or "unknown"): int(count) for value, count in rows}
@@ -2321,6 +2531,7 @@ CAMPAIGN_JOB_GRID_SORT_COLUMNS = {
"validation": CampaignJob.validation_status,
"queue": CampaignJob.queue_status,
"send": CampaignJob.send_status,
"postbox": CampaignJob.postbox_status,
"imap": CampaignJob.imap_status,
"attempts": CampaignJob.attempt_count,
"updated": CampaignJob.updated_at,
@@ -2329,6 +2540,10 @@ CAMPAIGN_JOB_GRID_LIST_FILTERS = {
"validation": (CampaignJob.validation_status, {item.value for item in JobValidationStatus}),
"queue": (CampaignJob.queue_status, {item.value for item in JobQueueStatus}),
"send": (CampaignJob.send_status, {item.value for item in JobSendStatus}),
"postbox": (
CampaignJob.postbox_status,
{item.value for item in JobPostboxStatus},
),
"imap": (CampaignJob.imap_status, {item.value for item in JobImapStatus}),
}
@@ -2643,13 +2858,14 @@ class CampaignJobsQuery:
validation_status: list[str] | None = Query(default=None),
imap_status: list[str] | None = Query(default=None),
query_text: str | None = Query(default=None, alias="q", max_length=200),
sort_by: Literal["number", "recipient", "subject", "validation", "queue", "send", "imap", "attempts", "updated"] = Query(default="number"),
sort_by: Literal["number", "recipient", "subject", "validation", "queue", "send", "postbox", "imap", "attempts", "updated"] = Query(default="number"),
sort_direction: Literal["asc", "desc"] = Query(default="asc"),
filter_recipient: str | None = Query(default=None, max_length=500),
filter_subject: str | None = Query(default=None, max_length=1000),
filter_validation: str | None = Query(default=None, max_length=1000),
filter_queue: str | None = Query(default=None, max_length=1000),
filter_send: str | None = Query(default=None, max_length=1000),
filter_postbox: str | None = Query(default=None, max_length=1000),
filter_imap: str | None = Query(default=None, max_length=1000),
filter_attempts: str | None = Query(default=None, max_length=100),
filter_evidence: str | None = Query(default=None, max_length=500),
@@ -2672,6 +2888,7 @@ class CampaignJobsQuery:
"validation": filter_validation,
"queue": filter_queue,
"send": filter_send,
"postbox": filter_postbox,
"imap": filter_imap,
"attempts": filter_attempts,
"evidence": filter_evidence,
@@ -2966,21 +3183,34 @@ def get_job_detail(
job = session.get(CampaignJob, job_id)
if not job or job.campaign_id != campaign.id or job.tenant_id != principal.tenant_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign job not found")
send_attempts = (
send_attempts = _job_attempt_rows(
session.query(SendAttempt)
.filter(SendAttempt.job_id == job.id)
.order_by(SendAttempt.attempt_number.asc())
.all()
.order_by(SendAttempt.attempt_number.asc()),
label="SMTP attempts for this campaign job",
)
imap_attempts = (
imap_attempts = _job_attempt_rows(
session.query(ImapAppendAttempt)
.filter(ImapAppendAttempt.job_id == job.id)
.order_by(ImapAppendAttempt.attempt_number.asc())
.all()
.order_by(ImapAppendAttempt.attempt_number.asc()),
label="IMAP attempts for this campaign job",
)
postbox_attempts = _job_attempt_rows(
session.query(PostboxDeliveryAttempt)
.filter(PostboxDeliveryAttempt.job_id == job.id)
.order_by(
PostboxDeliveryAttempt.target_index.asc(),
PostboxDeliveryAttempt.attempt_number.asc(),
),
label="Postbox attempts for this campaign job",
)
return CampaignJobDetailResponse(
job=_job_detail_payload(job),
attempts=_job_attempts_payload(send_attempts, imap_attempts),
attempts=_job_attempts_payload(
send_attempts,
imap_attempts,
postbox_attempts,
),
)
@@ -3002,19 +3232,33 @@ def get_job_diagnostics(
job = session.get(CampaignJob, job_id)
if not job or job.campaign_id != campaign.id or job.tenant_id != principal.tenant_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign job not found")
send_attempts = (
send_attempts = _job_attempt_rows(
session.query(SendAttempt)
.filter(SendAttempt.job_id == job.id)
.order_by(SendAttempt.attempt_number.asc())
.all()
.order_by(SendAttempt.attempt_number.asc()),
label="SMTP diagnostics for this campaign job",
)
imap_attempts = (
imap_attempts = _job_attempt_rows(
session.query(ImapAppendAttempt)
.filter(ImapAppendAttempt.job_id == job.id)
.order_by(ImapAppendAttempt.attempt_number.asc())
.all()
.order_by(ImapAppendAttempt.attempt_number.asc()),
label="IMAP diagnostics for this campaign job",
)
postbox_attempts = _job_attempt_rows(
session.query(PostboxDeliveryAttempt)
.filter(PostboxDeliveryAttempt.job_id == job.id)
.order_by(
PostboxDeliveryAttempt.target_index.asc(),
PostboxDeliveryAttempt.attempt_number.asc(),
),
label="Postbox diagnostics for this campaign job",
)
return _job_diagnostics_payload(
job,
send_attempts,
imap_attempts,
postbox_attempts,
)
return _job_diagnostics_payload(job, send_attempts, imap_attempts)
@router.get("/{campaign_id}/summary")
@@ -3156,6 +3400,7 @@ def email_campaign_report(
@router.get("/{campaign_id}/share-targets", response_model=CampaignShareTargetsResponse)
def list_campaign_share_targets(
campaign_id: str,
limit: int = Query(default=500, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:campaign:share")),
):
@@ -3163,6 +3408,14 @@ def list_campaign_share_targets(
directory = _access_directory()
users = [user for user in directory.users_for_tenant(principal.tenant_id) if user.status == "active"]
groups = [group for group in directory.groups_for_tenant(principal.tenant_id) if group.status == "active"]
if len(users) > limit or len(groups) > limit:
raise HTTPException(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
detail=(
f"Campaign share targets exceed the maximum response size of {limit} "
"users or groups. Use a searchable directory selector."
),
)
return CampaignShareTargetsResponse(
users=[CampaignShareTargetItem(id=item.id, name=item.display_name or item.email, secondary=item.email) for item in users],
groups=[CampaignShareTargetItem(id=item.id, name=item.name, secondary=None) for item in groups],
@@ -3172,17 +3425,35 @@ def list_campaign_share_targets(
@router.get("/{campaign_id}/shares", response_model=CampaignShareListResponse)
def list_campaign_shares(
campaign_id: str,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=500, ge=1, le=1000),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:campaign:share")),
):
campaign = _get_campaign_for_principal(session, campaign_id, principal, write=True)
shares = (
query = (
session.query(CampaignShare)
.filter(CampaignShare.tenant_id == principal.tenant_id, CampaignShare.campaign_id == campaign.id, CampaignShare.revoked_at.is_(None))
.order_by(CampaignShare.target_type.asc(), CampaignShare.target_id.asc())
)
total = query.order_by(None).count()
pages = max(1, (total + page_size - 1) // page_size)
shares = (
query.order_by(
CampaignShare.target_type.asc(),
CampaignShare.target_id.asc(),
CampaignShare.id.asc(),
)
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
return CampaignShareListResponse(shares=[CampaignShareItem.model_validate(item) for item in shares])
return CampaignShareListResponse(
shares=[CampaignShareItem.model_validate(item) for item in shares],
total=total,
page=page,
page_size=page_size,
pages=pages,
)
@router.put("/{campaign_id}/owner", response_model=CampaignResponse)
@@ -3306,7 +3577,8 @@ def campaign_delivery_options(
tenant_id=principal.tenant_id,
campaign_id=campaign_id,
version_id=version_id,
)
),
postbox_available=postbox_integration().available,
)
except QueueingError as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
@@ -3476,6 +3748,7 @@ def resolve_campaign_job_outcome(
job_id=job_id,
decision=payload.decision,
note=payload.note,
attempt_id=payload.attempt_id,
commit=False,
)
audit_from_principal(