Reconcile iCalendar replies from mail
This commit is contained in:
@@ -292,6 +292,17 @@ implemented.
|
||||
6. Record the incident/reconciliation reference in the consuming domain's audit
|
||||
trail without copying raw provider secrets or message content unnecessarily.
|
||||
|
||||
### Delivery-status and calendar-reply sources
|
||||
|
||||
An authorized Mail bounce source scans a bounded IMAP UID range without
|
||||
changing mailbox flags. It correlates DSN reports with durable Mail commands
|
||||
and, when the optional Calendar invitation capability is active, forwards
|
||||
`text/calendar` or `.ics` `METHOD:REPLY` parts to Calendar. Calendar remains
|
||||
owner of attendee state; Mail records only a raw digest, mailbox coordinates,
|
||||
Message-ID, and audit linkage. Ordinary or malformed calendar messages do not
|
||||
block DSN progress. A repeated UID/message digest produces no second Calendar
|
||||
state transition or outbound synchronization effect.
|
||||
|
||||
### Backup, restore, and retirement
|
||||
|
||||
Backups contain encrypted credentials and therefore need the same protection as
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from email import policy
|
||||
from email.message import EmailMessage
|
||||
@@ -14,6 +14,10 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.audit.logging import audit_event
|
||||
from govoplan_core.core.calendar import (
|
||||
CalendarCapabilityError,
|
||||
calendar_invitation_provider,
|
||||
)
|
||||
from govoplan_core.core.mail import (
|
||||
MailBounceObservationRef,
|
||||
MailBounceProcessingProvider,
|
||||
@@ -33,6 +37,7 @@ from govoplan_mail.backend.server_hierarchy import (
|
||||
hierarchy_context_for_profile,
|
||||
resolve_mail_transport,
|
||||
)
|
||||
from govoplan_mail.backend.runtime import get_registry
|
||||
|
||||
|
||||
class MailBounceError(RuntimeError):
|
||||
@@ -48,6 +53,96 @@ def normalize_message_id(value: object | None) -> str | None:
|
||||
return normalized[:998] or None
|
||||
|
||||
|
||||
def _calendar_reply_parts(raw_message: bytes) -> tuple[tuple[str, str | None], ...]:
|
||||
try:
|
||||
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||
except Exception as exc:
|
||||
raise MailBounceError("Mail message could not be parsed.") from exc
|
||||
parts: list[tuple[str, str | None]] = []
|
||||
for part in message.walk():
|
||||
filename = part.get_filename()
|
||||
if part.get_content_type() != "text/calendar" and not str(
|
||||
filename or ""
|
||||
).casefold().endswith(".ics"):
|
||||
continue
|
||||
try:
|
||||
content = part.get_content()
|
||||
except Exception:
|
||||
payload = part.get_payload(decode=True) or b""
|
||||
content = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
|
||||
if isinstance(content, bytes):
|
||||
content = content.decode(
|
||||
part.get_content_charset() or "utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
text = str(content).strip()
|
||||
if text:
|
||||
parts.append((text, filename))
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def _reconcile_calendar_replies(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
profile_id: str,
|
||||
folder: str,
|
||||
uid: str,
|
||||
raw_message: bytes,
|
||||
) -> int:
|
||||
provider = calendar_invitation_provider(get_registry())
|
||||
if provider is None:
|
||||
return 0
|
||||
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
|
||||
try:
|
||||
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||
message_id = normalize_message_id(message.get("Message-ID"))
|
||||
except Exception:
|
||||
message_id = None
|
||||
recorded_count = 0
|
||||
for icalendar, filename in _calendar_reply_parts(raw_message):
|
||||
try:
|
||||
recorded = provider.record_icalendar_reply(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
icalendar=icalendar,
|
||||
received_at=utcnow(),
|
||||
evidence={
|
||||
"transport": "mail-imap",
|
||||
"profile_id": profile_id,
|
||||
"folder": folder,
|
||||
"mailbox_uid": uid,
|
||||
"message_id": message_id,
|
||||
"filename": filename,
|
||||
"raw_sha256": raw_sha256,
|
||||
},
|
||||
)
|
||||
except CalendarCapabilityError:
|
||||
# Mailboxes routinely contain unrelated or malformed invitations.
|
||||
# They are not allowed to block DSN progress for the entire source.
|
||||
continue
|
||||
for invitation in recorded:
|
||||
audit_event(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
user_id=None,
|
||||
action="mail.calendar_reply.reconciled",
|
||||
object_type="calendar_invitation",
|
||||
object_id=invitation.event_id,
|
||||
details={
|
||||
"profile_id": profile_id,
|
||||
"folder": folder,
|
||||
"mailbox_uid": uid,
|
||||
"message_id": message_id,
|
||||
"calendar_uid": invitation.uid,
|
||||
"correlation_id": invitation.correlation_id,
|
||||
"raw_sha256": raw_sha256,
|
||||
},
|
||||
)
|
||||
recorded_count += 1
|
||||
return recorded_count
|
||||
|
||||
|
||||
def configure_bounce_source(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -154,7 +249,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
source = db.get(MailBounceSource, source_id)
|
||||
if source is None or source.tenant_id != tenant_id:
|
||||
raise MailBounceError("Bounce source not found.")
|
||||
processed, observations = self._scan_source(
|
||||
processed, observations, calendar_replies = self._scan_source(
|
||||
db,
|
||||
source,
|
||||
limit=max(1, min(int(limit), 1_000)),
|
||||
@@ -163,6 +258,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
"sources": 1,
|
||||
"processed_messages": processed,
|
||||
"observations": observations,
|
||||
"calendar_replies": calendar_replies,
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
@@ -175,8 +271,18 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
folder: str,
|
||||
uid: str,
|
||||
raw_message: bytes,
|
||||
reconcile_calendar: bool = True,
|
||||
) -> tuple[MailBounceObservationRef, ...]:
|
||||
db = _session(session)
|
||||
if reconcile_calendar:
|
||||
_reconcile_calendar_replies(
|
||||
db,
|
||||
tenant_id=tenant_id,
|
||||
profile_id=profile_id,
|
||||
folder=folder,
|
||||
uid=uid,
|
||||
raw_message=raw_message,
|
||||
)
|
||||
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
|
||||
reports = parse_delivery_status(raw_message)
|
||||
observations: list[MailBounceObservationRef] = []
|
||||
@@ -286,14 +392,20 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
)
|
||||
processed = 0
|
||||
observations = 0
|
||||
calendar_replies = 0
|
||||
failures: list[dict[str, str]] = []
|
||||
for source in sources:
|
||||
if remaining <= 0:
|
||||
break
|
||||
try:
|
||||
count, found = self._scan_source(db, source, limit=remaining)
|
||||
count, found, replies = self._scan_source(
|
||||
db,
|
||||
source,
|
||||
limit=remaining,
|
||||
)
|
||||
processed += count
|
||||
observations += found
|
||||
calendar_replies += replies
|
||||
remaining -= count
|
||||
except Exception as exc:
|
||||
source.last_scanned_at = utcnow()
|
||||
@@ -304,6 +416,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
"sources": len(sources),
|
||||
"processed_messages": processed,
|
||||
"observations": observations,
|
||||
"calendar_replies": calendar_replies,
|
||||
"failures": failures,
|
||||
}
|
||||
|
||||
@@ -336,7 +449,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
source: MailBounceSource,
|
||||
*,
|
||||
limit: int,
|
||||
) -> tuple[int, int]:
|
||||
) -> tuple[int, int, int]:
|
||||
profile = _profile(
|
||||
session,
|
||||
tenant_id=source.tenant_id,
|
||||
@@ -360,6 +473,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
limit=limit,
|
||||
)
|
||||
found = 0
|
||||
calendar_replies = 0
|
||||
highest = 0 if page.cursor_reset else source.highest_processed_uid
|
||||
for uid in page.uids:
|
||||
raw = get_imap_raw_message(
|
||||
@@ -367,6 +481,14 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
folder=source.folder,
|
||||
uid=uid,
|
||||
)
|
||||
calendar_replies += _reconcile_calendar_replies(
|
||||
session,
|
||||
tenant_id=source.tenant_id,
|
||||
profile_id=source.profile_id,
|
||||
folder=source.folder,
|
||||
uid=uid,
|
||||
raw_message=raw.raw,
|
||||
)
|
||||
found += len(
|
||||
self.process_raw_message(
|
||||
session,
|
||||
@@ -375,6 +497,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
folder=source.folder,
|
||||
uid=uid,
|
||||
raw_message=raw.raw,
|
||||
reconcile_calendar=False,
|
||||
)
|
||||
)
|
||||
highest = max(highest, int(uid))
|
||||
@@ -385,7 +508,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
||||
source.last_success_at = now
|
||||
source.last_error = None
|
||||
session.flush()
|
||||
return len(page.uids), found
|
||||
return len(page.uids), found, calendar_replies
|
||||
|
||||
|
||||
def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]:
|
||||
|
||||
@@ -299,7 +299,7 @@ manifest = ModuleManifest(
|
||||
name="Mail",
|
||||
version="0.1.10",
|
||||
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
|
||||
optional_dependencies=("campaigns", "addresses"),
|
||||
optional_dependencies=("campaigns", "addresses", "calendar"),
|
||||
provides_interfaces=(
|
||||
ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.2.0"),
|
||||
ModuleInterfaceProvider(name="mail.delivery_commands", version="0.1.0"),
|
||||
@@ -308,6 +308,12 @@ manifest = ModuleManifest(
|
||||
ModuleInterfaceProvider(name="mail.bounce_processing", version="0.1.0"),
|
||||
),
|
||||
requires_interfaces=(
|
||||
ModuleInterfaceRequirement(
|
||||
name="calendar.invitations",
|
||||
version_min="0.2.0",
|
||||
version_max_exclusive="0.3.0",
|
||||
optional=True,
|
||||
),
|
||||
ModuleInterfaceRequirement(
|
||||
name="campaigns.access",
|
||||
version_min="0.1.0",
|
||||
@@ -445,6 +451,9 @@ manifest = ModuleManifest(
|
||||
"and records idempotent per-recipient observations. SMTP acceptance remains "
|
||||
"separate from a later bounce. Unmatched reports remain visible for review; "
|
||||
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body. "
|
||||
"When Calendar is active, the same bounded source scan forwards METHOD:REPLY "
|
||||
"iCalendar parts to Calendar for idempotent attendee reconciliation without "
|
||||
"classifying the message as a bounce."
|
||||
),
|
||||
layer="configured",
|
||||
documentation_types=("admin", "user"),
|
||||
@@ -456,7 +465,7 @@ manifest = ModuleManifest(
|
||||
any_scopes=("mail:bounce:read", "mail:bounce:manage"),
|
||||
),
|
||||
),
|
||||
related_modules=("campaigns", "audit"),
|
||||
related_modules=("campaigns", "calendar", "audit"),
|
||||
metadata={
|
||||
"kind": "workflow",
|
||||
"route": "/mail/bounces",
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.core.calendar import CalendarInvitationRef
|
||||
from govoplan_mail.backend.bounce_processing import (
|
||||
SqlMailBounceProcessingProvider,
|
||||
parse_delivery_status,
|
||||
@@ -57,6 +58,24 @@ Body
|
||||
--dsn--
|
||||
"""
|
||||
|
||||
CALENDAR_REPLY = b"""From: Ada <ada@example.test>
|
||||
To: organizer@example.test
|
||||
Message-ID: <calendar-reply-1@example.test>
|
||||
Subject: Accepted: Planning
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/calendar; method=REPLY; charset=utf-8
|
||||
|
||||
BEGIN:VCALENDAR
|
||||
VERSION:2.0
|
||||
METHOD:REPLY
|
||||
BEGIN:VEVENT
|
||||
UID:invitation-1@govoplan.local
|
||||
DTSTART:20260805T090000Z
|
||||
ATTENDEE;PARTSTAT=ACCEPTED:mailto:ada@example.test
|
||||
END:VEVENT
|
||||
END:VCALENDAR
|
||||
"""
|
||||
|
||||
|
||||
class MailBounceProcessingTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
@@ -181,6 +200,49 @@ class MailBounceProcessingTests(unittest.TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_calendar_reply_is_forwarded_without_becoming_a_bounce(self) -> None:
|
||||
class CalendarProvider:
|
||||
def __init__(self) -> None:
|
||||
self.calls = []
|
||||
|
||||
def record_icalendar_reply(self, session, **kwargs):
|
||||
self.calls.append((session, kwargs))
|
||||
return (
|
||||
CalendarInvitationRef(
|
||||
event_id="event-1",
|
||||
calendar_id="calendar-1",
|
||||
uid="invitation-1@govoplan.local",
|
||||
correlation_id="campaign:version-1:entry-1",
|
||||
source_module="campaigns",
|
||||
source_resource_type="campaign_version",
|
||||
source_resource_id="version-1",
|
||||
),
|
||||
)
|
||||
|
||||
calendar = CalendarProvider()
|
||||
provider = SqlMailBounceProcessingProvider()
|
||||
with (
|
||||
self.SessionLocal() as session,
|
||||
patch(
|
||||
"govoplan_mail.backend.bounce_processing.calendar_invitation_provider",
|
||||
return_value=calendar,
|
||||
),
|
||||
):
|
||||
observations = provider.process_raw_message(
|
||||
session,
|
||||
tenant_id="tenant-1",
|
||||
profile_id="profile-1",
|
||||
folder="INBOX",
|
||||
uid="43",
|
||||
raw_message=CALENDAR_REPLY,
|
||||
)
|
||||
|
||||
self.assertEqual((), observations)
|
||||
self.assertEqual(1, len(calendar.calls))
|
||||
forwarded = calendar.calls[0][1]
|
||||
self.assertIn("METHOD:REPLY", forwarded["icalendar"])
|
||||
self.assertEqual("43", forwarded["evidence"]["mailbox_uid"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user