GovOPlaN/govoplan-mail
2
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Reconcile iCalendar replies from mail

Browse Source
This commit is contained in:
zemion
2026-08-02 16:38:45 +02:00
parent 5a3b13e26e
commit ff3aa066ae
4 changed files with 213 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/MAIL_HANDBOOK.md
+11
View File
@@ -292,6 +292,17 @@ implemented.
6. Record the incident/reconciliation reference in the consuming domain's audit
trail without copying raw provider secrets or message content unnecessarily.
### Delivery-status and calendar-reply sources
An authorized Mail bounce source scans a bounded IMAP UID range without
changing mailbox flags. It correlates DSN reports with durable Mail commands
and, when the optional Calendar invitation capability is active, forwards
`text/calendar` or `.ics` `METHOD:REPLY` parts to Calendar. Calendar remains
owner of attendee state; Mail records only a raw digest, mailbox coordinates,
Message-ID, and audit linkage. Ordinary or malformed calendar messages do not
block DSN progress. A repeated UID/message digest produces no second Calendar
state transition or outbound synchronization effect.
### Backup, restore, and retirement
Backups contain encrypted credentials and therefore need the same protection as
src/govoplan_mail/backend/bounce_processing.py
+128 -5
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import hashlib
from collections import defaultdict
from collections.abc import Mapping, Sequence
from collections.abc import Mapping
from datetime import datetime, timezone
from email import policy
from email.message import EmailMessage
@@ -14,6 +14,10 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.core.calendar import (
CalendarCapabilityError,
calendar_invitation_provider,
)
from govoplan_core.core.mail import (
MailBounceObservationRef,
MailBounceProcessingProvider,
@@ -33,6 +37,7 @@ from govoplan_mail.backend.server_hierarchy import (
hierarchy_context_for_profile,
resolve_mail_transport,
)
from govoplan_mail.backend.runtime import get_registry
class MailBounceError(RuntimeError):
@@ -48,6 +53,96 @@ def normalize_message_id(value: object | None) -> str | None:
return normalized[:998] or None
def _calendar_reply_parts(raw_message: bytes) -> tuple[tuple[str, str | None], ...]:
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
except Exception as exc:
raise MailBounceError("Mail message could not be parsed.") from exc
parts: list[tuple[str, str | None]] = []
for part in message.walk():
filename = part.get_filename()
if part.get_content_type() != "text/calendar" and not str(
filename or ""
).casefold().endswith(".ics"):
continue
try:
content = part.get_content()
except Exception:
payload = part.get_payload(decode=True) or b""
content = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
if isinstance(content, bytes):
content = content.decode(
part.get_content_charset() or "utf-8",
errors="replace",
)
text = str(content).strip()
if text:
parts.append((text, filename))
return tuple(parts)
def _reconcile_calendar_replies(
session: Session,
*,
tenant_id: str,
profile_id: str,
folder: str,
uid: str,
raw_message: bytes,
) -> int:
provider = calendar_invitation_provider(get_registry())
if provider is None:
return 0
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
try:
message = BytesParser(policy=policy.default).parsebytes(raw_message)
message_id = normalize_message_id(message.get("Message-ID"))
except Exception:
message_id = None
recorded_count = 0
for icalendar, filename in _calendar_reply_parts(raw_message):
try:
recorded = provider.record_icalendar_reply(
session,
tenant_id=tenant_id,
icalendar=icalendar,
received_at=utcnow(),
evidence={
"transport": "mail-imap",
"profile_id": profile_id,
"folder": folder,
"mailbox_uid": uid,
"message_id": message_id,
"filename": filename,
"raw_sha256": raw_sha256,
},
)
except CalendarCapabilityError:
# Mailboxes routinely contain unrelated or malformed invitations.
# They are not allowed to block DSN progress for the entire source.
continue
for invitation in recorded:
audit_event(
session,
tenant_id=tenant_id,
user_id=None,
action="mail.calendar_reply.reconciled",
object_type="calendar_invitation",
object_id=invitation.event_id,
details={
"profile_id": profile_id,
"folder": folder,
"mailbox_uid": uid,
"message_id": message_id,
"calendar_uid": invitation.uid,
"correlation_id": invitation.correlation_id,
"raw_sha256": raw_sha256,
},
)
recorded_count += 1
return recorded_count
def configure_bounce_source(
session: Session,
*,
@@ -154,7 +249,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
source = db.get(MailBounceSource, source_id)
if source is None or source.tenant_id != tenant_id:
raise MailBounceError("Bounce source not found.")
processed, observations = self._scan_source(
processed, observations, calendar_replies = self._scan_source(
db,
source,
limit=max(1, min(int(limit), 1_000)),
@@ -163,6 +258,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
"sources": 1,
"processed_messages": processed,
"observations": observations,
"calendar_replies": calendar_replies,
"failures": [],
}
@@ -175,8 +271,18 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
folder: str,
uid: str,
raw_message: bytes,
reconcile_calendar: bool = True,
) -> tuple[MailBounceObservationRef, ...]:
db = _session(session)
if reconcile_calendar:
_reconcile_calendar_replies(
db,
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
uid=uid,
raw_message=raw_message,
)
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
reports = parse_delivery_status(raw_message)
observations: list[MailBounceObservationRef] = []
@@ -286,14 +392,20 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
)
processed = 0
observations = 0
calendar_replies = 0
failures: list[dict[str, str]] = []
for source in sources:
if remaining <= 0:
break
try:
count, found = self._scan_source(db, source, limit=remaining)
count, found, replies = self._scan_source(
db,
source,
limit=remaining,
)
processed += count
observations += found
calendar_replies += replies
remaining -= count
except Exception as exc:
source.last_scanned_at = utcnow()
@@ -304,6 +416,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
"sources": len(sources),
"processed_messages": processed,
"observations": observations,
"calendar_replies": calendar_replies,
"failures": failures,
}
@@ -336,7 +449,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
source: MailBounceSource,
*,
limit: int,
) -> tuple[int, int]:
) -> tuple[int, int, int]:
profile = _profile(
session,
tenant_id=source.tenant_id,
@@ -360,6 +473,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
limit=limit,
)
found = 0
calendar_replies = 0
highest = 0 if page.cursor_reset else source.highest_processed_uid
for uid in page.uids:
raw = get_imap_raw_message(
@@ -367,6 +481,14 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
folder=source.folder,
uid=uid,
)
calendar_replies += _reconcile_calendar_replies(
session,
tenant_id=source.tenant_id,
profile_id=source.profile_id,
folder=source.folder,
uid=uid,
raw_message=raw.raw,
)
found += len(
self.process_raw_message(
session,
@@ -375,6 +497,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
folder=source.folder,
uid=uid,
raw_message=raw.raw,
reconcile_calendar=False,
)
)
highest = max(highest, int(uid))
@@ -385,7 +508,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
source.last_success_at = now
source.last_error = None
session.flush()
return len(page.uids), found
return len(page.uids), found, calendar_replies
def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]:
src/govoplan_mail/backend/manifest.py
+12 -3
View File
@@ -299,7 +299,7 @@ manifest = ModuleManifest(
name="Mail",
version="0.1.10",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("campaigns", "addresses"),
optional_dependencies=("campaigns", "addresses", "calendar"),
provides_interfaces=(
ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.2.0"),
ModuleInterfaceProvider(name="mail.delivery_commands", version="0.1.0"),
@@ -308,6 +308,12 @@ manifest = ModuleManifest(
ModuleInterfaceProvider(name="mail.bounce_processing", version="0.1.0"),
),
requires_interfaces=(
ModuleInterfaceRequirement(
name="calendar.invitations",
version_min="0.2.0",
version_max_exclusive="0.3.0",
optional=True,
),
ModuleInterfaceRequirement(
name="campaigns.access",
version_min="0.1.0",
@@ -444,7 +450,10 @@ manifest = ModuleManifest(
"bounded message/delivery-status reports without changing mailbox flags, "
"and records idempotent per-recipient observations. SMTP acceptance remains "
"separate from a later bounce. Unmatched reports remain visible for review; "
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body."
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body. "
"When Calendar is active, the same bounded source scan forwards METHOD:REPLY "
"iCalendar parts to Calendar for idempotent attendee reconciliation without "
"classifying the message as a bounce."
),
layer="configured",
documentation_types=("admin", "user"),
@@ -456,7 +465,7 @@ manifest = ModuleManifest(
any_scopes=("mail:bounce:read", "mail:bounce:manage"),
),
),
related_modules=("campaigns", "audit"),
related_modules=("campaigns", "calendar", "audit"),
metadata={
"kind": "workflow",
"route": "/mail/bounces",
tests/test_bounce_processing.py
+62
View File
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from govoplan_core.db.base import Base
from govoplan_core.core.calendar import CalendarInvitationRef
from govoplan_mail.backend.bounce_processing import (
SqlMailBounceProcessingProvider,
parse_delivery_status,
@@ -57,6 +58,24 @@ Body
--dsn--
"""
CALENDAR_REPLY = b"""From: Ada <ada@example.test>
To: organizer@example.test
Message-ID: <calendar-reply-1@example.test>
Subject: Accepted: Planning
MIME-Version: 1.0
Content-Type: text/calendar; method=REPLY; charset=utf-8
BEGIN:VCALENDAR
VERSION:2.0
METHOD:REPLY
BEGIN:VEVENT
UID:invitation-1@govoplan.local
DTSTART:20260805T090000Z
ATTENDEE;PARTSTAT=ACCEPTED:mailto:ada@example.test
END:VEVENT
END:VCALENDAR
"""
class MailBounceProcessingTests(unittest.TestCase):
def setUp(self) -> None:
@@ -181,6 +200,49 @@ class MailBounceProcessingTests(unittest.TestCase):
),
)
def test_calendar_reply_is_forwarded_without_becoming_a_bounce(self) -> None:
class CalendarProvider:
def __init__(self) -> None:
self.calls = []
def record_icalendar_reply(self, session, **kwargs):
self.calls.append((session, kwargs))
return (
CalendarInvitationRef(
event_id="event-1",
calendar_id="calendar-1",
uid="invitation-1@govoplan.local",
correlation_id="campaign:version-1:entry-1",
source_module="campaigns",
source_resource_type="campaign_version",
source_resource_id="version-1",
),
)
calendar = CalendarProvider()
provider = SqlMailBounceProcessingProvider()
with (
self.SessionLocal() as session,
patch(
"govoplan_mail.backend.bounce_processing.calendar_invitation_provider",
return_value=calendar,
),
):
observations = provider.process_raw_message(
session,
tenant_id="tenant-1",
profile_id="profile-1",
folder="INBOX",
uid="43",
raw_message=CALENDAR_REPLY,
)
self.assertEqual((), observations)
self.assertEqual(1, len(calendar.calls))
forwarded = calendar.calls[0][1]
self.assertIn("METHOD:REPLY", forwarded["icalendar"])
self.assertEqual("43", forwarded["evidence"]["mailbox_uid"])
if __name__ == "__main__":
unittest.main()