diff --git a/docs/MAIL_HANDBOOK.md b/docs/MAIL_HANDBOOK.md index 208aa89..1570d5a 100644 --- a/docs/MAIL_HANDBOOK.md +++ b/docs/MAIL_HANDBOOK.md @@ -292,6 +292,17 @@ implemented. 6. Record the incident/reconciliation reference in the consuming domain's audit trail without copying raw provider secrets or message content unnecessarily. +### Delivery-status and calendar-reply sources + +An authorized Mail bounce source scans a bounded IMAP UID range without +changing mailbox flags. It correlates DSN reports with durable Mail commands +and, when the optional Calendar invitation capability is active, forwards +`text/calendar` or `.ics` `METHOD:REPLY` parts to Calendar. Calendar remains +owner of attendee state; Mail records only a raw digest, mailbox coordinates, +Message-ID, and audit linkage. Ordinary or malformed calendar messages do not +block DSN progress. A repeated UID/message digest produces no second Calendar +state transition or outbound synchronization effect. + ### Backup, restore, and retirement Backups contain encrypted credentials and therefore need the same protection as diff --git a/src/govoplan_mail/backend/bounce_processing.py b/src/govoplan_mail/backend/bounce_processing.py index 822ff49..f017e11 100644 --- a/src/govoplan_mail/backend/bounce_processing.py +++ b/src/govoplan_mail/backend/bounce_processing.py @@ -2,7 +2,7 @@ from __future__ import annotations import hashlib from collections import defaultdict -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from datetime import datetime, timezone from email import policy from email.message import EmailMessage @@ -14,6 +14,10 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from govoplan_core.audit.logging import audit_event +from govoplan_core.core.calendar import ( + CalendarCapabilityError, + calendar_invitation_provider, +) from govoplan_core.core.mail import ( MailBounceObservationRef, MailBounceProcessingProvider, @@ -33,6 +37,7 @@ from govoplan_mail.backend.server_hierarchy import ( hierarchy_context_for_profile, resolve_mail_transport, ) +from govoplan_mail.backend.runtime import get_registry class MailBounceError(RuntimeError): @@ -48,6 +53,96 @@ def normalize_message_id(value: object | None) -> str | None: return normalized[:998] or None +def _calendar_reply_parts(raw_message: bytes) -> tuple[tuple[str, str | None], ...]: + try: + message = BytesParser(policy=policy.default).parsebytes(raw_message) + except Exception as exc: + raise MailBounceError("Mail message could not be parsed.") from exc + parts: list[tuple[str, str | None]] = [] + for part in message.walk(): + filename = part.get_filename() + if part.get_content_type() != "text/calendar" and not str( + filename or "" + ).casefold().endswith(".ics"): + continue + try: + content = part.get_content() + except Exception: + payload = part.get_payload(decode=True) or b"" + content = payload.decode(part.get_content_charset() or "utf-8", errors="replace") + if isinstance(content, bytes): + content = content.decode( + part.get_content_charset() or "utf-8", + errors="replace", + ) + text = str(content).strip() + if text: + parts.append((text, filename)) + return tuple(parts) + + +def _reconcile_calendar_replies( + session: Session, + *, + tenant_id: str, + profile_id: str, + folder: str, + uid: str, + raw_message: bytes, +) -> int: + provider = calendar_invitation_provider(get_registry()) + if provider is None: + return 0 + raw_sha256 = hashlib.sha256(raw_message).hexdigest() + try: + message = BytesParser(policy=policy.default).parsebytes(raw_message) + message_id = normalize_message_id(message.get("Message-ID")) + except Exception: + message_id = None + recorded_count = 0 + for icalendar, filename in _calendar_reply_parts(raw_message): + try: + recorded = provider.record_icalendar_reply( + session, + tenant_id=tenant_id, + icalendar=icalendar, + received_at=utcnow(), + evidence={ + "transport": "mail-imap", + "profile_id": profile_id, + "folder": folder, + "mailbox_uid": uid, + "message_id": message_id, + "filename": filename, + "raw_sha256": raw_sha256, + }, + ) + except CalendarCapabilityError: + # Mailboxes routinely contain unrelated or malformed invitations. + # They are not allowed to block DSN progress for the entire source. + continue + for invitation in recorded: + audit_event( + session, + tenant_id=tenant_id, + user_id=None, + action="mail.calendar_reply.reconciled", + object_type="calendar_invitation", + object_id=invitation.event_id, + details={ + "profile_id": profile_id, + "folder": folder, + "mailbox_uid": uid, + "message_id": message_id, + "calendar_uid": invitation.uid, + "correlation_id": invitation.correlation_id, + "raw_sha256": raw_sha256, + }, + ) + recorded_count += 1 + return recorded_count + + def configure_bounce_source( session: Session, *, @@ -154,7 +249,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): source = db.get(MailBounceSource, source_id) if source is None or source.tenant_id != tenant_id: raise MailBounceError("Bounce source not found.") - processed, observations = self._scan_source( + processed, observations, calendar_replies = self._scan_source( db, source, limit=max(1, min(int(limit), 1_000)), @@ -163,6 +258,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): "sources": 1, "processed_messages": processed, "observations": observations, + "calendar_replies": calendar_replies, "failures": [], } @@ -175,8 +271,18 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): folder: str, uid: str, raw_message: bytes, + reconcile_calendar: bool = True, ) -> tuple[MailBounceObservationRef, ...]: db = _session(session) + if reconcile_calendar: + _reconcile_calendar_replies( + db, + tenant_id=tenant_id, + profile_id=profile_id, + folder=folder, + uid=uid, + raw_message=raw_message, + ) raw_sha256 = hashlib.sha256(raw_message).hexdigest() reports = parse_delivery_status(raw_message) observations: list[MailBounceObservationRef] = [] @@ -286,14 +392,20 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): ) processed = 0 observations = 0 + calendar_replies = 0 failures: list[dict[str, str]] = [] for source in sources: if remaining <= 0: break try: - count, found = self._scan_source(db, source, limit=remaining) + count, found, replies = self._scan_source( + db, + source, + limit=remaining, + ) processed += count observations += found + calendar_replies += replies remaining -= count except Exception as exc: source.last_scanned_at = utcnow() @@ -304,6 +416,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): "sources": len(sources), "processed_messages": processed, "observations": observations, + "calendar_replies": calendar_replies, "failures": failures, } @@ -336,7 +449,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): source: MailBounceSource, *, limit: int, - ) -> tuple[int, int]: + ) -> tuple[int, int, int]: profile = _profile( session, tenant_id=source.tenant_id, @@ -360,6 +473,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): limit=limit, ) found = 0 + calendar_replies = 0 highest = 0 if page.cursor_reset else source.highest_processed_uid for uid in page.uids: raw = get_imap_raw_message( @@ -367,6 +481,14 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): folder=source.folder, uid=uid, ) + calendar_replies += _reconcile_calendar_replies( + session, + tenant_id=source.tenant_id, + profile_id=source.profile_id, + folder=source.folder, + uid=uid, + raw_message=raw.raw, + ) found += len( self.process_raw_message( session, @@ -375,6 +497,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): folder=source.folder, uid=uid, raw_message=raw.raw, + reconcile_calendar=False, ) ) highest = max(highest, int(uid)) @@ -385,7 +508,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider): source.last_success_at = now source.last_error = None session.flush() - return len(page.uids), found + return len(page.uids), found, calendar_replies def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]: diff --git a/src/govoplan_mail/backend/manifest.py b/src/govoplan_mail/backend/manifest.py index f016272..fb448f4 100644 --- a/src/govoplan_mail/backend/manifest.py +++ b/src/govoplan_mail/backend/manifest.py @@ -299,7 +299,7 @@ manifest = ModuleManifest( name="Mail", version="0.1.10", required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR), - optional_dependencies=("campaigns", "addresses"), + optional_dependencies=("campaigns", "addresses", "calendar"), provides_interfaces=( ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.2.0"), ModuleInterfaceProvider(name="mail.delivery_commands", version="0.1.0"), @@ -308,6 +308,12 @@ manifest = ModuleManifest( ModuleInterfaceProvider(name="mail.bounce_processing", version="0.1.0"), ), requires_interfaces=( + ModuleInterfaceRequirement( + name="calendar.invitations", + version_min="0.2.0", + version_max_exclusive="0.3.0", + optional=True, + ), ModuleInterfaceRequirement( name="campaigns.access", version_min="0.1.0", @@ -444,7 +450,10 @@ manifest = ModuleManifest( "bounded message/delivery-status reports without changing mailbox flags, " "and records idempotent per-recipient observations. SMTP acceptance remains " "separate from a later bounce. Unmatched reports remain visible for review; " - "Mail stores only bounded diagnostics and a raw digest, not the raw bounce body." + "Mail stores only bounded diagnostics and a raw digest, not the raw bounce body. " + "When Calendar is active, the same bounded source scan forwards METHOD:REPLY " + "iCalendar parts to Calendar for idempotent attendee reconciliation without " + "classifying the message as a bounce." ), layer="configured", documentation_types=("admin", "user"), @@ -456,7 +465,7 @@ manifest = ModuleManifest( any_scopes=("mail:bounce:read", "mail:bounce:manage"), ), ), - related_modules=("campaigns", "audit"), + related_modules=("campaigns", "calendar", "audit"), metadata={ "kind": "workflow", "route": "/mail/bounces", diff --git a/tests/test_bounce_processing.py b/tests/test_bounce_processing.py index 906b49a..72e4dcd 100644 --- a/tests/test_bounce_processing.py +++ b/tests/test_bounce_processing.py @@ -8,6 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import StaticPool from govoplan_core.db.base import Base +from govoplan_core.core.calendar import CalendarInvitationRef from govoplan_mail.backend.bounce_processing import ( SqlMailBounceProcessingProvider, parse_delivery_status, @@ -57,6 +58,24 @@ Body --dsn-- """ +CALENDAR_REPLY = b"""From: Ada +To: organizer@example.test +Message-ID: +Subject: Accepted: Planning +MIME-Version: 1.0 +Content-Type: text/calendar; method=REPLY; charset=utf-8 + +BEGIN:VCALENDAR +VERSION:2.0 +METHOD:REPLY +BEGIN:VEVENT +UID:invitation-1@govoplan.local +DTSTART:20260805T090000Z +ATTENDEE;PARTSTAT=ACCEPTED:mailto:ada@example.test +END:VEVENT +END:VCALENDAR +""" + class MailBounceProcessingTests(unittest.TestCase): def setUp(self) -> None: @@ -181,6 +200,49 @@ class MailBounceProcessingTests(unittest.TestCase): ), ) + def test_calendar_reply_is_forwarded_without_becoming_a_bounce(self) -> None: + class CalendarProvider: + def __init__(self) -> None: + self.calls = [] + + def record_icalendar_reply(self, session, **kwargs): + self.calls.append((session, kwargs)) + return ( + CalendarInvitationRef( + event_id="event-1", + calendar_id="calendar-1", + uid="invitation-1@govoplan.local", + correlation_id="campaign:version-1:entry-1", + source_module="campaigns", + source_resource_type="campaign_version", + source_resource_id="version-1", + ), + ) + + calendar = CalendarProvider() + provider = SqlMailBounceProcessingProvider() + with ( + self.SessionLocal() as session, + patch( + "govoplan_mail.backend.bounce_processing.calendar_invitation_provider", + return_value=calendar, + ), + ): + observations = provider.process_raw_message( + session, + tenant_id="tenant-1", + profile_id="profile-1", + folder="INBOX", + uid="43", + raw_message=CALENDAR_REPLY, + ) + + self.assertEqual((), observations) + self.assertEqual(1, len(calendar.calls)) + forwarded = calendar.calls[0][1] + self.assertIn("METHOD:REPLY", forwarded["icalendar"]) + self.assertEqual("43", forwarded["evidence"]["mailbox_uid"]) + if __name__ == "__main__": unittest.main()