545 lines
17 KiB
Python
545 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict
|
|
from sqlalchemy import case, func, or_
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion
|
|
from govoplan_campaign.backend.report_privacy_policy import (
|
|
CampaignReportPrivacyPolicy,
|
|
effective_campaign_report_privacy_policy,
|
|
)
|
|
|
|
|
|
class AggregateCampaignReportError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class AggregateReportCampaign(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
id: str
|
|
name: str
|
|
status: str
|
|
|
|
|
|
class AggregateReportCampaignListItem(AggregateReportCampaign):
|
|
updated_at: datetime
|
|
|
|
|
|
class AggregateReportCampaignList(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
campaigns: list[AggregateReportCampaignListItem]
|
|
|
|
|
|
class AggregateCount(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
value: int | None
|
|
suppressed: bool = False
|
|
|
|
|
|
class AggregatePopulation(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
denominator: AggregateCount
|
|
denominator_definition: str
|
|
inactive_source_entries: AggregateCount
|
|
excluded_or_blocked_jobs: AggregateCount
|
|
|
|
|
|
class AggregateOutcomeCounts(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
smtp_accepted: AggregateCount
|
|
postbox_accepted: AggregateCount
|
|
delivered: AggregateCount
|
|
partially_accepted: AggregateCount
|
|
failed: AggregateCount
|
|
outcome_unknown: AggregateCount
|
|
queued_or_active: AggregateCount
|
|
cancelled: AggregateCount
|
|
excluded: AggregateCount
|
|
not_attempted: AggregateCount
|
|
|
|
|
|
class AggregateTimeRange(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
first_activity_at: datetime | None
|
|
last_activity_at: datetime | None
|
|
suppressed: bool
|
|
|
|
|
|
class AggregatePrivacy(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
small_cell_threshold: int
|
|
suppression_applied: bool
|
|
rule: str
|
|
|
|
|
|
class AggregateCampaignReport(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
generated_at: datetime
|
|
campaign: AggregateReportCampaign
|
|
version_number: int | None
|
|
completion_state: Literal[
|
|
"not_started",
|
|
"in_progress",
|
|
"completed",
|
|
"partially_completed",
|
|
"incomplete",
|
|
"outcome_unknown",
|
|
"suppressed",
|
|
]
|
|
population: AggregatePopulation
|
|
outcomes: AggregateOutcomeCounts
|
|
time_range: AggregateTimeRange
|
|
privacy: AggregatePrivacy
|
|
|
|
|
|
_OUTCOME_KEYS = (
|
|
"smtp_accepted",
|
|
"postbox_accepted",
|
|
"delivered",
|
|
"partially_accepted",
|
|
"failed",
|
|
"outcome_unknown",
|
|
"queued_or_active",
|
|
"cancelled",
|
|
"excluded",
|
|
"not_attempted",
|
|
)
|
|
|
|
|
|
def generate_aggregate_campaign_report(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version_id: str | None = None,
|
|
) -> AggregateCampaignReport:
|
|
"""Build the deliberately small, recipient-free Campaign report projection."""
|
|
|
|
campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
|
|
version = _selected_version(session, campaign, version_id)
|
|
facts = _query_aggregate_facts(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
campaign_id=campaign.id,
|
|
version=version,
|
|
)
|
|
policy = effective_campaign_report_privacy_policy(session, tenant_id=tenant_id)
|
|
return _build_aggregate_campaign_report(
|
|
campaign=campaign,
|
|
version=version,
|
|
facts=facts,
|
|
policy=policy,
|
|
)
|
|
|
|
|
|
def _get_campaign(session: Session, *, tenant_id: str, campaign_id: str) -> Campaign:
|
|
campaign = (
|
|
session.query(Campaign)
|
|
.filter(Campaign.tenant_id == tenant_id, Campaign.id == campaign_id)
|
|
.one_or_none()
|
|
)
|
|
if campaign is None:
|
|
raise AggregateCampaignReportError("Campaign not found")
|
|
return campaign
|
|
|
|
|
|
def _selected_version(
|
|
session: Session,
|
|
campaign: Campaign,
|
|
version_id: str | None,
|
|
) -> CampaignVersion | None:
|
|
selected_id = version_id or campaign.current_version_id
|
|
if not selected_id:
|
|
return None
|
|
version = session.get(CampaignVersion, selected_id)
|
|
if version is None or version.campaign_id != campaign.id:
|
|
raise AggregateCampaignReportError("Campaign version not found")
|
|
return version
|
|
|
|
|
|
def _query_aggregate_facts(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
campaign_id: str,
|
|
version: CampaignVersion | None,
|
|
) -> _AggregateFacts:
|
|
if version is None:
|
|
return _AggregateFacts.empty()
|
|
|
|
smtp_accepted = CampaignJob.send_status.in_(
|
|
{"smtp_accepted", "sent"}
|
|
)
|
|
accepted = CampaignJob.send_status.in_(
|
|
{
|
|
"smtp_accepted",
|
|
"postbox_accepted",
|
|
"delivered",
|
|
"partially_accepted",
|
|
"sent",
|
|
}
|
|
)
|
|
failed = CampaignJob.send_status.in_({"failed_temporary", "failed_permanent"})
|
|
unknown = CampaignJob.send_status == "outcome_unknown"
|
|
active = CampaignJob.send_status.in_({"queued", "claimed", "sending"})
|
|
cancelled = CampaignJob.send_status == "cancelled"
|
|
excluded = CampaignJob.send_status == "skipped"
|
|
excluded_or_blocked = or_(
|
|
CampaignJob.validation_status.in_({"blocked", "excluded", "inactive"}),
|
|
CampaignJob.build_status != "built",
|
|
)
|
|
row = (
|
|
session.query(
|
|
func.count(CampaignJob.id).label("denominator"),
|
|
func.sum(case((smtp_accepted, 1), else_=0)).label(
|
|
"smtp_accepted"
|
|
),
|
|
func.sum(
|
|
case(
|
|
(
|
|
CampaignJob.send_status == "postbox_accepted",
|
|
1,
|
|
),
|
|
else_=0,
|
|
)
|
|
).label("postbox_accepted"),
|
|
func.sum(
|
|
case(
|
|
(CampaignJob.send_status == "delivered", 1),
|
|
else_=0,
|
|
)
|
|
).label("delivered"),
|
|
func.sum(
|
|
case(
|
|
(
|
|
CampaignJob.send_status == "partially_accepted",
|
|
1,
|
|
),
|
|
else_=0,
|
|
)
|
|
).label("partially_accepted"),
|
|
func.sum(case((failed, 1), else_=0)).label("failed"),
|
|
func.sum(case((unknown, 1), else_=0)).label("outcome_unknown"),
|
|
func.sum(case((active, 1), else_=0)).label("queued_or_active"),
|
|
func.sum(case((cancelled, 1), else_=0)).label("cancelled"),
|
|
func.sum(case((excluded, 1), else_=0)).label("excluded"),
|
|
func.sum(
|
|
case((or_(accepted, failed, unknown, active, cancelled, excluded), 0), else_=1)
|
|
).label("not_attempted"),
|
|
func.sum(case((excluded_or_blocked, 1), else_=0)).label("excluded_or_blocked"),
|
|
func.min(CampaignJob.queued_at).label("queued_min"),
|
|
func.max(CampaignJob.queued_at).label("queued_max"),
|
|
func.min(CampaignJob.smtp_started_at).label("smtp_min"),
|
|
func.max(CampaignJob.smtp_started_at).label("smtp_max"),
|
|
func.min(CampaignJob.sent_at).label("sent_min"),
|
|
func.max(CampaignJob.sent_at).label("sent_max"),
|
|
func.min(CampaignJob.outcome_unknown_at).label("unknown_min"),
|
|
func.max(CampaignJob.outcome_unknown_at).label("unknown_max"),
|
|
)
|
|
.filter(
|
|
CampaignJob.tenant_id == tenant_id,
|
|
CampaignJob.campaign_id == campaign_id,
|
|
CampaignJob.campaign_version_id == version.id,
|
|
)
|
|
.one()
|
|
)
|
|
values = row._mapping
|
|
activity = [
|
|
values[key]
|
|
for key in (
|
|
"queued_min",
|
|
"queued_max",
|
|
"smtp_min",
|
|
"smtp_max",
|
|
"sent_min",
|
|
"sent_max",
|
|
"unknown_min",
|
|
"unknown_max",
|
|
)
|
|
if values[key] is not None
|
|
]
|
|
return _AggregateFacts(
|
|
outcomes={key: int(values[key] or 0) for key in _OUTCOME_KEYS},
|
|
denominator=int(values["denominator"] or 0),
|
|
excluded_or_blocked=int(values["excluded_or_blocked"] or 0),
|
|
first_activity_at=min(activity) if activity else None,
|
|
last_activity_at=max(activity) if activity else None,
|
|
)
|
|
|
|
|
|
def build_aggregate_campaign_report(
|
|
*,
|
|
campaign: Campaign,
|
|
version: CampaignVersion | None,
|
|
jobs: list[CampaignJob],
|
|
policy: CampaignReportPrivacyPolicy,
|
|
generated_at: datetime | None = None,
|
|
) -> AggregateCampaignReport:
|
|
return _build_aggregate_campaign_report(
|
|
campaign=campaign,
|
|
version=version,
|
|
facts=_facts_from_jobs(jobs),
|
|
policy=policy,
|
|
generated_at=generated_at,
|
|
)
|
|
|
|
|
|
def _build_aggregate_campaign_report(
|
|
*,
|
|
campaign: Campaign,
|
|
version: CampaignVersion | None,
|
|
facts: _AggregateFacts,
|
|
policy: CampaignReportPrivacyPolicy,
|
|
generated_at: datetime | None = None,
|
|
) -> AggregateCampaignReport:
|
|
outcome_values = facts.outcomes
|
|
outcomes, denominator = _suppress_partition(
|
|
outcome_values,
|
|
threshold=policy.small_cell_threshold,
|
|
)
|
|
outcome_suppression_applied = denominator.suppressed or any(
|
|
item.suppressed for item in outcomes.values()
|
|
)
|
|
inactive_entries = _inactive_entry_count(version)
|
|
standalone_counts = {
|
|
"inactive_source_entries": _suppress_standalone_count(
|
|
inactive_entries,
|
|
threshold=policy.small_cell_threshold,
|
|
),
|
|
# This population count overlaps the outcome partition, so exposing it
|
|
# can make a suppressed outcome recoverable through subtraction.
|
|
"excluded_or_blocked_jobs": (
|
|
AggregateCount(value=None, suppressed=True)
|
|
if outcome_suppression_applied
|
|
else _suppress_standalone_count(
|
|
facts.excluded_or_blocked,
|
|
threshold=policy.small_cell_threshold,
|
|
)
|
|
),
|
|
}
|
|
suppression_applied = denominator.suppressed or any(
|
|
item.suppressed for item in (*outcomes.values(), *standalone_counts.values())
|
|
)
|
|
first_activity = facts.first_activity_at
|
|
last_activity = facts.last_activity_at
|
|
suppress_time_range = suppression_applied or (
|
|
0 < facts.denominator < policy.small_cell_threshold
|
|
)
|
|
|
|
return AggregateCampaignReport(
|
|
generated_at=generated_at or datetime.now(timezone.utc),
|
|
campaign=AggregateReportCampaign(
|
|
id=campaign.id,
|
|
name=campaign.name,
|
|
status=campaign.status,
|
|
),
|
|
version_number=version.version_number if version else None,
|
|
completion_state=(
|
|
"suppressed"
|
|
if denominator.suppressed
|
|
else _completion_state(outcome_values, facts.denominator)
|
|
),
|
|
population=AggregatePopulation(
|
|
denominator=denominator,
|
|
denominator_definition=(
|
|
"All persisted recipient delivery jobs for the selected campaign version, "
|
|
"including excluded or blocked jobs. Inactive source entries without a job "
|
|
"record are excluded and reported separately."
|
|
),
|
|
inactive_source_entries=standalone_counts["inactive_source_entries"],
|
|
excluded_or_blocked_jobs=standalone_counts["excluded_or_blocked_jobs"],
|
|
),
|
|
outcomes=AggregateOutcomeCounts(**outcomes),
|
|
time_range=AggregateTimeRange(
|
|
first_activity_at=None if suppress_time_range else first_activity,
|
|
last_activity_at=None if suppress_time_range else last_activity,
|
|
suppressed=suppress_time_range and first_activity is not None,
|
|
),
|
|
privacy=AggregatePrivacy(
|
|
small_cell_threshold=policy.small_cell_threshold,
|
|
suppression_applied=suppression_applied,
|
|
rule=(
|
|
"Positive counts below the threshold are hidden. At least one additional "
|
|
"count or the denominator is hidden when needed to prevent subtraction. "
|
|
"Overlapping population counts are hidden whenever outcome suppression applies."
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
def aggregate_report_campaign_item(campaign: Campaign) -> AggregateReportCampaignListItem:
|
|
return AggregateReportCampaignListItem(
|
|
id=campaign.id,
|
|
name=campaign.name,
|
|
status=campaign.status,
|
|
updated_at=campaign.updated_at,
|
|
)
|
|
|
|
|
|
def _outcome_counts(jobs: list[CampaignJob]) -> dict[str, int]:
|
|
counts: Counter[str] = Counter()
|
|
for job in jobs:
|
|
status = job.send_status
|
|
if status in {"smtp_accepted", "sent"}:
|
|
counts["smtp_accepted"] += 1
|
|
elif status == "postbox_accepted":
|
|
counts["postbox_accepted"] += 1
|
|
elif status == "delivered":
|
|
counts["delivered"] += 1
|
|
elif status == "partially_accepted":
|
|
counts["partially_accepted"] += 1
|
|
elif status in {"failed_temporary", "failed_permanent"}:
|
|
counts["failed"] += 1
|
|
elif status == "outcome_unknown":
|
|
counts["outcome_unknown"] += 1
|
|
elif status in {"queued", "claimed", "sending"}:
|
|
counts["queued_or_active"] += 1
|
|
elif status == "cancelled":
|
|
counts["cancelled"] += 1
|
|
elif status == "skipped":
|
|
counts["excluded"] += 1
|
|
else:
|
|
counts["not_attempted"] += 1
|
|
return {key: counts[key] for key in _OUTCOME_KEYS}
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _AggregateFacts:
|
|
outcomes: dict[str, int]
|
|
denominator: int
|
|
excluded_or_blocked: int
|
|
first_activity_at: datetime | None
|
|
last_activity_at: datetime | None
|
|
|
|
@classmethod
|
|
def empty(cls) -> _AggregateFacts:
|
|
return cls(
|
|
outcomes={key: 0 for key in _OUTCOME_KEYS},
|
|
denominator=0,
|
|
excluded_or_blocked=0,
|
|
first_activity_at=None,
|
|
last_activity_at=None,
|
|
)
|
|
|
|
|
|
def _facts_from_jobs(jobs: list[CampaignJob]) -> _AggregateFacts:
|
|
first_activity, last_activity = _activity_range(jobs)
|
|
return _AggregateFacts(
|
|
outcomes=_outcome_counts(jobs),
|
|
denominator=len(jobs),
|
|
excluded_or_blocked=sum(
|
|
1
|
|
for job in jobs
|
|
if job.validation_status in {"blocked", "excluded", "inactive"}
|
|
or job.build_status != "built"
|
|
),
|
|
first_activity_at=first_activity,
|
|
last_activity_at=last_activity,
|
|
)
|
|
|
|
|
|
def _suppress_partition(
|
|
counts: dict[str, int],
|
|
*,
|
|
threshold: int,
|
|
) -> tuple[dict[str, AggregateCount], AggregateCount]:
|
|
total = sum(counts.values())
|
|
suppressed = {key for key, value in counts.items() if 0 < value < threshold}
|
|
suppress_denominator = False
|
|
if suppressed:
|
|
companions = [
|
|
(value, key)
|
|
for key, value in counts.items()
|
|
if key not in suppressed and value > 0
|
|
]
|
|
if companions:
|
|
suppressed.add(max(companions)[1])
|
|
else:
|
|
suppress_denominator = True
|
|
denominator = AggregateCount(
|
|
value=None if suppress_denominator else total,
|
|
suppressed=suppress_denominator,
|
|
)
|
|
return (
|
|
{
|
|
key: AggregateCount(
|
|
value=None if key in suppressed else value,
|
|
suppressed=key in suppressed,
|
|
)
|
|
for key, value in counts.items()
|
|
},
|
|
denominator,
|
|
)
|
|
|
|
|
|
def _suppress_standalone_count(value: int, *, threshold: int) -> AggregateCount:
|
|
if 0 < value < threshold:
|
|
return AggregateCount(value=None, suppressed=True)
|
|
return AggregateCount(value=value, suppressed=False)
|
|
|
|
|
|
def _completion_state(
|
|
counts: dict[str, int],
|
|
total: int,
|
|
) -> Literal[
|
|
"not_started",
|
|
"in_progress",
|
|
"completed",
|
|
"partially_completed",
|
|
"incomplete",
|
|
"outcome_unknown",
|
|
]:
|
|
if total == 0 or counts["not_attempted"] + counts["excluded"] == total:
|
|
return "not_started"
|
|
if counts["outcome_unknown"]:
|
|
return "outcome_unknown"
|
|
if counts["queued_or_active"]:
|
|
return "in_progress"
|
|
fully_accepted = (
|
|
counts["smtp_accepted"]
|
|
+ counts["postbox_accepted"]
|
|
+ counts["delivered"]
|
|
)
|
|
partially_accepted = counts["partially_accepted"]
|
|
if fully_accepted == total:
|
|
return "completed"
|
|
if fully_accepted or partially_accepted:
|
|
return "partially_completed"
|
|
return "incomplete"
|
|
|
|
|
|
def _activity_range(jobs: list[CampaignJob]) -> tuple[datetime | None, datetime | None]:
|
|
activity = [
|
|
value
|
|
for job in jobs
|
|
for value in (
|
|
job.queued_at,
|
|
job.smtp_started_at,
|
|
job.sent_at,
|
|
job.outcome_unknown_at,
|
|
)
|
|
if value is not None
|
|
]
|
|
if not activity:
|
|
return None, None
|
|
return min(activity), max(activity)
|
|
|
|
|
|
def _inactive_entry_count(version: CampaignVersion | None) -> int:
|
|
build_summary = version.build_summary if version and isinstance(version.build_summary, dict) else {}
|
|
return int(build_summary.get("inactive_count") or build_summary.get("inactive_entries_count") or 0)
|