507 lines
16 KiB
Python
507 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from email import policy
|
|
from email.parser import BytesParser
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.core.postbox import (
|
|
PostboxAttachmentRef,
|
|
PostboxDeliveryOutcomeUnknown,
|
|
PostboxDeliveryRejected,
|
|
PostboxDeliveryRequest,
|
|
PostboxParticipantRef,
|
|
PostboxTargetRef,
|
|
)
|
|
from govoplan_campaign.backend.db.models import (
|
|
CampaignJob,
|
|
JobPostboxStatus,
|
|
PostboxDeliveryAttempt,
|
|
)
|
|
from govoplan_campaign.backend.integrations import (
|
|
PostboxDeliveryUnavailable,
|
|
postbox_integration,
|
|
)
|
|
from govoplan_core.security.time import utc_now
|
|
|
|
|
|
ACCEPTED_POSTBOX_ATTEMPT_STATUSES = {
|
|
JobPostboxStatus.ACCEPTED.value,
|
|
JobPostboxStatus.ACCEPTED_VACANT.value,
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class PostboxChannelOutcome:
|
|
accepted: int = 0
|
|
accepted_vacant: int = 0
|
|
rejected_temporary: int = 0
|
|
rejected_permanent: int = 0
|
|
outcome_unknown: int = 0
|
|
messages: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def accepted_count(self) -> int:
|
|
return self.accepted + self.accepted_vacant
|
|
|
|
@property
|
|
def rejected_count(self) -> int:
|
|
return self.rejected_temporary + self.rejected_permanent
|
|
|
|
@property
|
|
def all_rejected_before_acceptance(self) -> bool:
|
|
return (
|
|
self.accepted_count == 0
|
|
and self.outcome_unknown == 0
|
|
and self.rejected_count > 0
|
|
)
|
|
|
|
@property
|
|
def status(self) -> str:
|
|
if self.outcome_unknown:
|
|
return JobPostboxStatus.OUTCOME_UNKNOWN.value
|
|
if self.accepted_count and self.rejected_count:
|
|
return JobPostboxStatus.PARTIALLY_ACCEPTED.value
|
|
if self.accepted_vacant and not self.accepted:
|
|
return JobPostboxStatus.ACCEPTED_VACANT.value
|
|
if self.accepted_count:
|
|
return JobPostboxStatus.ACCEPTED.value
|
|
if self.rejected_temporary:
|
|
return JobPostboxStatus.REJECTED_TEMPORARY.value
|
|
return JobPostboxStatus.REJECTED_PERMANENT.value
|
|
|
|
|
|
def _target_key(target: dict[str, Any]) -> str:
|
|
payload = json.dumps(
|
|
target,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
default=str,
|
|
).encode("utf-8")
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def _attempt_number(
|
|
session: Session,
|
|
*,
|
|
job_id: str,
|
|
target_key: str,
|
|
) -> int:
|
|
current = (
|
|
session.query(func.max(PostboxDeliveryAttempt.attempt_number))
|
|
.filter(
|
|
PostboxDeliveryAttempt.job_id == job_id,
|
|
PostboxDeliveryAttempt.target_key == target_key,
|
|
)
|
|
.scalar()
|
|
)
|
|
return int(current or 0) + 1
|
|
|
|
|
|
def _accepted_attempt(
|
|
session: Session,
|
|
*,
|
|
job_id: str,
|
|
target_key: str,
|
|
) -> PostboxDeliveryAttempt | None:
|
|
return (
|
|
session.query(PostboxDeliveryAttempt)
|
|
.filter(
|
|
PostboxDeliveryAttempt.job_id == job_id,
|
|
PostboxDeliveryAttempt.target_key == target_key,
|
|
PostboxDeliveryAttempt.status.in_(
|
|
list(ACCEPTED_POSTBOX_ATTEMPT_STATUSES)
|
|
),
|
|
)
|
|
.order_by(PostboxDeliveryAttempt.attempt_number.desc())
|
|
.first()
|
|
)
|
|
|
|
|
|
def _message_body(message_bytes: bytes) -> str | None:
|
|
message = BytesParser(policy=policy.default).parsebytes(message_bytes)
|
|
body = message.get_body(preferencelist=("plain", "html"))
|
|
if body is None:
|
|
payload = message.get_payload(decode=True)
|
|
if not isinstance(payload, bytes):
|
|
return None
|
|
return payload.decode(message.get_content_charset() or "utf-8", "replace")
|
|
try:
|
|
content = body.get_content()
|
|
except (LookupError, UnicodeError):
|
|
payload = body.get_payload(decode=True)
|
|
if not isinstance(payload, bytes):
|
|
return None
|
|
return payload.decode(body.get_content_charset() or "utf-8", "replace")
|
|
return str(content)
|
|
|
|
|
|
def _participants(job: CampaignJob) -> tuple[PostboxParticipantRef, ...]:
|
|
recipients = job.resolved_recipients or {}
|
|
values: list[PostboxParticipantRef] = []
|
|
for key in (
|
|
"from_all",
|
|
"to",
|
|
"cc",
|
|
"bcc",
|
|
"reply_to",
|
|
"bounce_to",
|
|
"disposition_notification_to",
|
|
):
|
|
for item in recipients.get(key) or []:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
address = str(item.get("email") or "").strip() or None
|
|
label = str(item.get("name") or "").strip() or None
|
|
if address is None and label is None:
|
|
continue
|
|
values.append(
|
|
PostboxParticipantRef(
|
|
kind="sender" if key == "from_all" else key,
|
|
reference_type="mail_address",
|
|
label=label,
|
|
address=address,
|
|
)
|
|
)
|
|
return tuple(values)
|
|
|
|
|
|
def _attachments(job: CampaignJob) -> tuple[PostboxAttachmentRef, ...]:
|
|
values = [_eml_attachment(job)]
|
|
for rule_index, rule in enumerate(job.resolved_attachments or []):
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
managed_matches = rule.get("managed_matches")
|
|
if isinstance(managed_matches, list) and managed_matches:
|
|
values.extend(_managed_attachments(managed_matches))
|
|
continue
|
|
matches = rule.get("matches")
|
|
if isinstance(matches, list):
|
|
values.extend(_campaign_attachments(job, rule_index=rule_index, matches=matches))
|
|
return tuple(values)
|
|
|
|
|
|
def _eml_attachment(job: CampaignJob) -> PostboxAttachmentRef:
|
|
return PostboxAttachmentRef(
|
|
reference_type="campaign_eml",
|
|
reference_id=job.id,
|
|
name=f"{job.entry_id or job.entry_index}.eml",
|
|
media_type="message/rfc822",
|
|
size_bytes=job.eml_size_bytes,
|
|
digest=job.eml_sha256,
|
|
metadata={"campaign_id": job.campaign_id, "campaign_version_id": job.campaign_version_id},
|
|
)
|
|
|
|
|
|
MANAGED_ATTACHMENT_METADATA_KEYS = {
|
|
"asset_id",
|
|
"version_id",
|
|
"blob_id",
|
|
"display_path",
|
|
"relative_path",
|
|
"owner_type",
|
|
"owner_id",
|
|
"source_revision",
|
|
}
|
|
|
|
|
|
def _managed_attachment(match: dict[str, Any]) -> PostboxAttachmentRef | None:
|
|
reference_id = str(match.get("version_id") or match.get("asset_id") or match.get("blob_id") or "").strip()
|
|
if not reference_id:
|
|
return None
|
|
return PostboxAttachmentRef(
|
|
reference_type="file_version" if match.get("version_id") else "file_asset",
|
|
reference_id=reference_id,
|
|
name=str(match.get("filename") or "").strip() or None,
|
|
media_type=str(match.get("content_type")) if match.get("content_type") else None,
|
|
size_bytes=int(match["size_bytes"]) if match.get("size_bytes") is not None else None,
|
|
digest=str(match.get("checksum_sha256")) if match.get("checksum_sha256") else None,
|
|
metadata={key: value for key, value in match.items() if key in MANAGED_ATTACHMENT_METADATA_KEYS},
|
|
)
|
|
|
|
|
|
def _managed_attachments(matches: list[Any]) -> list[PostboxAttachmentRef]:
|
|
attachments = (_managed_attachment(match) for match in matches if isinstance(match, dict))
|
|
return [attachment for attachment in attachments if attachment is not None]
|
|
|
|
|
|
def _campaign_attachments(
|
|
job: CampaignJob,
|
|
*,
|
|
rule_index: int,
|
|
matches: list[Any],
|
|
) -> list[PostboxAttachmentRef]:
|
|
metadata = {"campaign_id": job.campaign_id, "campaign_version_id": job.campaign_version_id, "job_id": job.id}
|
|
return [
|
|
PostboxAttachmentRef(
|
|
reference_type="campaign_attachment",
|
|
reference_id=f"{job.id}:{rule_index}:{match_index}",
|
|
name=str(match).rsplit("/", 1)[-1] or None,
|
|
metadata=metadata,
|
|
)
|
|
for match_index, match in enumerate(matches)
|
|
]
|
|
|
|
|
|
def _sender_label(job: CampaignJob) -> str | None:
|
|
recipients = job.resolved_recipients or {}
|
|
sender = recipients.get("from")
|
|
if not isinstance(sender, dict):
|
|
return None
|
|
name = str(sender.get("name") or "").strip()
|
|
address = str(sender.get("email") or "").strip()
|
|
if name and address:
|
|
return f"{name} <{address}>"
|
|
return address or name or None
|
|
|
|
|
|
def _request(
|
|
job: CampaignJob,
|
|
target: dict[str, Any],
|
|
*,
|
|
target_key: str,
|
|
body_text: str | None,
|
|
classification: str,
|
|
) -> PostboxDeliveryRequest:
|
|
return PostboxDeliveryRequest(
|
|
tenant_id=job.tenant_id,
|
|
target=PostboxTargetRef(postbox_id=str(target["postbox_id"])),
|
|
producer_module="campaigns",
|
|
producer_resource_type="campaign_job",
|
|
producer_resource_id=job.id,
|
|
idempotency_key=(
|
|
f"campaign:{job.campaign_version_id}:{job.id}:postbox:"
|
|
f"{target_key}"
|
|
),
|
|
subject=(job.subject or "").strip() or "(No subject)",
|
|
body_text=body_text,
|
|
sender_label=_sender_label(job),
|
|
classification=classification,
|
|
participants=_participants(job),
|
|
attachments=_attachments(job),
|
|
metadata={
|
|
"campaign_id": job.campaign_id,
|
|
"campaign_version_id": job.campaign_version_id,
|
|
"campaign_job_id": job.id,
|
|
"entry_id": job.entry_id,
|
|
"entry_index": job.entry_index,
|
|
"delivery_channel_policy": job.delivery_channel_policy,
|
|
"target_snapshot": target,
|
|
},
|
|
)
|
|
|
|
|
|
def _record_rejection(
|
|
session: Session,
|
|
*,
|
|
job_id: str,
|
|
attempt_id: str,
|
|
exc: Exception,
|
|
temporary: bool,
|
|
) -> None:
|
|
session.rollback()
|
|
attempt = session.get(PostboxDeliveryAttempt, attempt_id)
|
|
job = session.get(CampaignJob, job_id)
|
|
if attempt is None or job is None:
|
|
raise RuntimeError(
|
|
"Postbox rejection could not be written to Campaign evidence."
|
|
) from exc
|
|
attempt.status = (
|
|
JobPostboxStatus.REJECTED_TEMPORARY.value
|
|
if temporary
|
|
else JobPostboxStatus.REJECTED_PERMANENT.value
|
|
)
|
|
attempt.error_type = exc.__class__.__name__
|
|
attempt.error_code = str(getattr(exc, "code", "") or "") or None
|
|
attempt.error_message = str(exc)
|
|
attempt.finished_at = utc_now()
|
|
session.add(attempt)
|
|
session.add(job)
|
|
session.commit()
|
|
|
|
|
|
def _record_unknown(
|
|
session: Session,
|
|
*,
|
|
job_id: str,
|
|
attempt_id: str,
|
|
exc: Exception,
|
|
) -> None:
|
|
session.rollback()
|
|
attempt = session.get(PostboxDeliveryAttempt, attempt_id)
|
|
job = session.get(CampaignJob, job_id)
|
|
if attempt is None or job is None:
|
|
raise RuntimeError(
|
|
"Unknown Postbox outcome could not be written to Campaign evidence."
|
|
) from exc
|
|
attempt.status = JobPostboxStatus.OUTCOME_UNKNOWN.value
|
|
attempt.error_type = exc.__class__.__name__
|
|
attempt.error_code = str(getattr(exc, "code", "") or "") or None
|
|
attempt.error_message = str(exc)
|
|
attempt.finished_at = utc_now()
|
|
job.postbox_status = JobPostboxStatus.OUTCOME_UNKNOWN.value
|
|
session.add(attempt)
|
|
session.add(job)
|
|
session.commit()
|
|
|
|
|
|
def deliver_campaign_job_to_postboxes(
|
|
session: Session,
|
|
*,
|
|
job: CampaignJob,
|
|
message_bytes: bytes,
|
|
classification: str,
|
|
) -> PostboxChannelOutcome:
|
|
outcome = PostboxChannelOutcome()
|
|
targets = [
|
|
target
|
|
for target in (job.resolved_postbox_targets or [])
|
|
if isinstance(target, dict) and target.get("postbox_id")
|
|
]
|
|
if not targets:
|
|
outcome.rejected_permanent = 1
|
|
outcome.messages.append("No frozen Postbox target is available.")
|
|
job.postbox_status = JobPostboxStatus.REJECTED_PERMANENT.value
|
|
session.add(job)
|
|
session.commit()
|
|
return outcome
|
|
|
|
body_text = _message_body(message_bytes)
|
|
for target_index, target in enumerate(targets):
|
|
key = _target_key(target)
|
|
accepted = _accepted_attempt(
|
|
session,
|
|
job_id=job.id,
|
|
target_key=key,
|
|
)
|
|
if accepted is not None:
|
|
if accepted.vacant:
|
|
outcome.accepted_vacant += 1
|
|
else:
|
|
outcome.accepted += 1
|
|
continue
|
|
|
|
attempt_number = _attempt_number(
|
|
session,
|
|
job_id=job.id,
|
|
target_key=key,
|
|
)
|
|
request = _request(
|
|
job,
|
|
target,
|
|
target_key=key,
|
|
body_text=body_text,
|
|
classification=classification,
|
|
)
|
|
attempt = PostboxDeliveryAttempt(
|
|
tenant_id=job.tenant_id,
|
|
job_id=job.id,
|
|
target_key=key,
|
|
target_index=target_index,
|
|
attempt_number=attempt_number,
|
|
idempotency_key=request.idempotency_key,
|
|
status=JobPostboxStatus.DELIVERING.value,
|
|
target_snapshot=target,
|
|
evidence={},
|
|
started_at=utc_now(),
|
|
)
|
|
job.postbox_attempt_count += 1
|
|
job.postbox_status = JobPostboxStatus.DELIVERING.value
|
|
session.add(attempt)
|
|
session.add(job)
|
|
session.commit()
|
|
attempt_id = attempt.id
|
|
|
|
try:
|
|
result = postbox_integration().deliver(session, request)
|
|
current_attempt = session.get(PostboxDeliveryAttempt, attempt_id)
|
|
current_job = session.get(CampaignJob, job.id)
|
|
if current_attempt is None or current_job is None:
|
|
raise RuntimeError(
|
|
"Campaign Postbox attempt disappeared before acceptance."
|
|
)
|
|
current_attempt.status = (
|
|
JobPostboxStatus.ACCEPTED_VACANT.value
|
|
if result.vacant
|
|
else JobPostboxStatus.ACCEPTED.value
|
|
)
|
|
current_attempt.provider_delivery_id = result.delivery_id
|
|
current_attempt.provider_message_id = result.message_id
|
|
current_attempt.postbox_id = result.postbox_id
|
|
current_attempt.address = result.address
|
|
current_attempt.holder_count = result.holder_count
|
|
current_attempt.vacant = result.vacant
|
|
current_attempt.duplicate = result.duplicate
|
|
current_attempt.evidence = dict(result.evidence)
|
|
current_attempt.finished_at = utc_now()
|
|
session.add(current_attempt)
|
|
session.add(current_job)
|
|
session.commit()
|
|
if result.vacant:
|
|
outcome.accepted_vacant += 1
|
|
else:
|
|
outcome.accepted += 1
|
|
except PostboxDeliveryOutcomeUnknown as exc:
|
|
_record_unknown(
|
|
session,
|
|
job_id=job.id,
|
|
attempt_id=attempt_id,
|
|
exc=exc,
|
|
)
|
|
outcome.outcome_unknown += 1
|
|
outcome.messages.append(str(exc))
|
|
except PostboxDeliveryRejected as exc:
|
|
if exc.code == "idempotency_conflict":
|
|
_record_unknown(
|
|
session,
|
|
job_id=job.id,
|
|
attempt_id=attempt_id,
|
|
exc=exc,
|
|
)
|
|
outcome.outcome_unknown += 1
|
|
else:
|
|
_record_rejection(
|
|
session,
|
|
job_id=job.id,
|
|
attempt_id=attempt_id,
|
|
exc=exc,
|
|
temporary=exc.temporary,
|
|
)
|
|
if exc.temporary:
|
|
outcome.rejected_temporary += 1
|
|
else:
|
|
outcome.rejected_permanent += 1
|
|
outcome.messages.append(str(exc))
|
|
except PostboxDeliveryUnavailable as exc:
|
|
_record_rejection(
|
|
session,
|
|
job_id=job.id,
|
|
attempt_id=attempt_id,
|
|
exc=exc,
|
|
temporary=True,
|
|
)
|
|
outcome.rejected_temporary += 1
|
|
outcome.messages.append(str(exc))
|
|
except Exception as exc:
|
|
_record_unknown(
|
|
session,
|
|
job_id=job.id,
|
|
attempt_id=attempt_id,
|
|
exc=exc,
|
|
)
|
|
outcome.outcome_unknown += 1
|
|
outcome.messages.append(str(exc))
|
|
|
|
current_job = session.get(CampaignJob, job.id)
|
|
if current_job is not None:
|
|
current_job.postbox_status = outcome.status
|
|
session.add(current_job)
|
|
session.commit()
|
|
return outcome
|