GovOPlaN/govoplan-mail
2
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Reconcile iCalendar replies from mail

Browse Source
This commit is contained in:
zemion
2026-08-02 16:38:45 +02:00
parent 5a3b13e26e
commit ff3aa066ae
4 changed files with 213 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_mail/backend/bounce_processing.py
+128 -5
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import hashlib
from collections import defaultdict
from collections.abc import Mapping, Sequence
from collections.abc import Mapping
from datetime import datetime, timezone
from email import policy
from email.message import EmailMessage
@@ -14,6 +14,10 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.core.calendar import (
CalendarCapabilityError,
calendar_invitation_provider,
)
from govoplan_core.core.mail import (
MailBounceObservationRef,
MailBounceProcessingProvider,
@@ -33,6 +37,7 @@ from govoplan_mail.backend.server_hierarchy import (
hierarchy_context_for_profile,
resolve_mail_transport,
)
from govoplan_mail.backend.runtime import get_registry
class MailBounceError(RuntimeError):
@@ -48,6 +53,96 @@ def normalize_message_id(value: object | None) -> str | None:
return normalized[:998] or None
def _calendar_reply_parts(raw_message: bytes) -> tuple[tuple[str, str | None], ...]:
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
except Exception as exc:
raise MailBounceError("Mail message could not be parsed.") from exc
parts: list[tuple[str, str | None]] = []
for part in message.walk():
filename = part.get_filename()
if part.get_content_type() != "text/calendar" and not str(
filename or ""
).casefold().endswith(".ics"):
continue
try:
content = part.get_content()
except Exception:
payload = part.get_payload(decode=True) or b""
content = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
if isinstance(content, bytes):
content = content.decode(
part.get_content_charset() or "utf-8",
errors="replace",
)
text = str(content).strip()
if text:
parts.append((text, filename))
return tuple(parts)
def _reconcile_calendar_replies(
session: Session,
*,
tenant_id: str,
profile_id: str,
folder: str,
uid: str,
raw_message: bytes,
) -> int:
provider = calendar_invitation_provider(get_registry())
if provider is None:
return 0
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
message_id = normalize_message_id(message.get("Message-ID"))
except Exception:
message_id = None
recorded_count = 0
for icalendar, filename in _calendar_reply_parts(raw_message):
try:
recorded = provider.record_icalendar_reply(
session,
tenant_id=tenant_id,
icalendar=icalendar,
received_at=utcnow(),
evidence={
"transport": "mail-imap",
"profile_id": profile_id,
"folder": folder,
"mailbox_uid": uid,
"message_id": message_id,
"filename": filename,
"raw_sha256": raw_sha256,
},
)
except CalendarCapabilityError:
# Mailboxes routinely contain unrelated or malformed invitations.
# They are not allowed to block DSN progress for the entire source.
continue
for invitation in recorded:
audit_event(
session,
tenant_id=tenant_id,
user_id=None,
action="mail.calendar_reply.reconciled",
object_type="calendar_invitation",
object_id=invitation.event_id,
details={
"profile_id": profile_id,
"folder": folder,
"mailbox_uid": uid,
"message_id": message_id,
"calendar_uid": invitation.uid,
"correlation_id": invitation.correlation_id,
"raw_sha256": raw_sha256,
},
)
recorded_count += 1
return recorded_count
def configure_bounce_source(
session: Session,
*,
@@ -154,7 +249,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
source = db.get(MailBounceSource, source_id)
if source is None or source.tenant_id != tenant_id:
raise MailBounceError("Bounce source not found.")
processed, observations = self._scan_source(
processed, observations, calendar_replies = self._scan_source(
db,
source,
limit=max(1, min(int(limit), 1_000)),
@@ -163,6 +258,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
"sources": 1,
"processed_messages": processed,
"observations": observations,
"calendar_replies": calendar_replies,
"failures": [],
}
@@ -175,8 +271,18 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
folder: str,
uid: str,
raw_message: bytes,
reconcile_calendar: bool = True,
) -> tuple[MailBounceObservationRef, ...]:
db = _session(session)
if reconcile_calendar:
_reconcile_calendar_replies(
db,
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
uid=uid,
raw_message=raw_message,
)
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
reports = parse_delivery_status(raw_message)
observations: list[MailBounceObservationRef] = []
@@ -286,14 +392,20 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
)
processed = 0
observations = 0
calendar_replies = 0
failures: list[dict[str, str]] = []
for source in sources:
if remaining <= 0:
break
try:
count, found = self._scan_source(db, source, limit=remaining)
count, found, replies = self._scan_source(
db,
source,
limit=remaining,
)
processed += count
observations += found
calendar_replies += replies
remaining -= count
except Exception as exc:
source.last_scanned_at = utcnow()
@@ -304,6 +416,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
"sources": len(sources),
"processed_messages": processed,
"observations": observations,
"calendar_replies": calendar_replies,
"failures": failures,
}
@@ -336,7 +449,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
source: MailBounceSource,
*,
limit: int,
) -> tuple[int, int]:
) -> tuple[int, int, int]:
profile = _profile(
session,
tenant_id=source.tenant_id,
@@ -360,6 +473,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
limit=limit,
)
found = 0
calendar_replies = 0
highest = 0 if page.cursor_reset else source.highest_processed_uid
for uid in page.uids:
raw = get_imap_raw_message(
@@ -367,6 +481,14 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
folder=source.folder,
uid=uid,
)
calendar_replies += _reconcile_calendar_replies(
session,
tenant_id=source.tenant_id,
profile_id=source.profile_id,
folder=source.folder,
uid=uid,
raw_message=raw.raw,
)
found += len(
self.process_raw_message(
session,
@@ -375,6 +497,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
folder=source.folder,
uid=uid,
raw_message=raw.raw,
reconcile_calendar=False,
)
)
highest = max(highest, int(uid))
@@ -385,7 +508,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
source.last_success_at = now
source.last_error = None
session.flush()
return len(page.uids), found
return len(page.uids), found, calendar_replies
def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]:
src/govoplan_mail/backend/manifest.py
+12 -3
View File
@@ -299,7 +299,7 @@ manifest = ModuleManifest(
name="Mail",
version="0.1.10",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("campaigns", "addresses"),
optional_dependencies=("campaigns", "addresses", "calendar"),
provides_interfaces=(
ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.2.0"),
ModuleInterfaceProvider(name="mail.delivery_commands", version="0.1.0"),
@@ -308,6 +308,12 @@ manifest = ModuleManifest(
ModuleInterfaceProvider(name="mail.bounce_processing", version="0.1.0"),
),
requires_interfaces=(
ModuleInterfaceRequirement(
name="calendar.invitations",
version_min="0.2.0",
version_max_exclusive="0.3.0",
optional=True,
),
ModuleInterfaceRequirement(
name="campaigns.access",
version_min="0.1.0",
@@ -444,7 +450,10 @@ manifest = ModuleManifest(
"bounded message/delivery-status reports without changing mailbox flags, "
"and records idempotent per-recipient observations. SMTP acceptance remains "
"separate from a later bounce. Unmatched reports remain visible for review; "
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body."
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body. "
"When Calendar is active, the same bounded source scan forwards METHOD:REPLY "
"iCalendar parts to Calendar for idempotent attendee reconciliation without "
"classifying the message as a bounce."
),
layer="configured",
documentation_types=("admin", "user"),
@@ -456,7 +465,7 @@ manifest = ModuleManifest(
any_scopes=("mail:bounce:read", "mail:bounce:manage"),
),
),
related_modules=("campaigns", "audit"),
related_modules=("campaigns", "calendar", "audit"),
metadata={
"kind": "workflow",
"route": "/mail/bounces",