Reconcile iCalendar replies from mail
This commit is contained in:
@@ -292,6 +292,17 @@ implemented.
|
|||||||
6. Record the incident/reconciliation reference in the consuming domain's audit
|
6. Record the incident/reconciliation reference in the consuming domain's audit
|
||||||
trail without copying raw provider secrets or message content unnecessarily.
|
trail without copying raw provider secrets or message content unnecessarily.
|
||||||
|
|
||||||
|
### Delivery-status and calendar-reply sources
|
||||||
|
|
||||||
|
An authorized Mail bounce source scans a bounded IMAP UID range without
|
||||||
|
changing mailbox flags. It correlates DSN reports with durable Mail commands
|
||||||
|
and, when the optional Calendar invitation capability is active, forwards
|
||||||
|
`text/calendar` or `.ics` `METHOD:REPLY` parts to Calendar. Calendar remains
|
||||||
|
owner of attendee state; Mail records only a raw digest, mailbox coordinates,
|
||||||
|
Message-ID, and audit linkage. Ordinary or malformed calendar messages do not
|
||||||
|
block DSN progress. A repeated UID/message digest produces no second Calendar
|
||||||
|
state transition or outbound synchronization effect.
|
||||||
|
|
||||||
### Backup, restore, and retirement
|
### Backup, restore, and retirement
|
||||||
|
|
||||||
Backups contain encrypted credentials and therefore need the same protection as
|
Backups contain encrypted credentials and therefore need the same protection as
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from email import policy
|
from email import policy
|
||||||
from email.message import EmailMessage
|
from email.message import EmailMessage
|
||||||
@@ -14,6 +14,10 @@ from sqlalchemy.exc import IntegrityError
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from govoplan_core.audit.logging import audit_event
|
from govoplan_core.audit.logging import audit_event
|
||||||
|
from govoplan_core.core.calendar import (
|
||||||
|
CalendarCapabilityError,
|
||||||
|
calendar_invitation_provider,
|
||||||
|
)
|
||||||
from govoplan_core.core.mail import (
|
from govoplan_core.core.mail import (
|
||||||
MailBounceObservationRef,
|
MailBounceObservationRef,
|
||||||
MailBounceProcessingProvider,
|
MailBounceProcessingProvider,
|
||||||
@@ -33,6 +37,7 @@ from govoplan_mail.backend.server_hierarchy import (
|
|||||||
hierarchy_context_for_profile,
|
hierarchy_context_for_profile,
|
||||||
resolve_mail_transport,
|
resolve_mail_transport,
|
||||||
)
|
)
|
||||||
|
from govoplan_mail.backend.runtime import get_registry
|
||||||
|
|
||||||
|
|
||||||
class MailBounceError(RuntimeError):
|
class MailBounceError(RuntimeError):
|
||||||
@@ -48,6 +53,96 @@ def normalize_message_id(value: object | None) -> str | None:
|
|||||||
return normalized[:998] or None
|
return normalized[:998] or None
|
||||||
|
|
||||||
|
|
||||||
|
def _calendar_reply_parts(raw_message: bytes) -> tuple[tuple[str, str | None], ...]:
|
||||||
|
try:
|
||||||
|
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||||
|
except Exception as exc:
|
||||||
|
raise MailBounceError("Mail message could not be parsed.") from exc
|
||||||
|
parts: list[tuple[str, str | None]] = []
|
||||||
|
for part in message.walk():
|
||||||
|
filename = part.get_filename()
|
||||||
|
if part.get_content_type() != "text/calendar" and not str(
|
||||||
|
filename or ""
|
||||||
|
).casefold().endswith(".ics"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
content = part.get_content()
|
||||||
|
except Exception:
|
||||||
|
payload = part.get_payload(decode=True) or b""
|
||||||
|
content = payload.decode(part.get_content_charset() or "utf-8", errors="replace")
|
||||||
|
if isinstance(content, bytes):
|
||||||
|
content = content.decode(
|
||||||
|
part.get_content_charset() or "utf-8",
|
||||||
|
errors="replace",
|
||||||
|
)
|
||||||
|
text = str(content).strip()
|
||||||
|
if text:
|
||||||
|
parts.append((text, filename))
|
||||||
|
return tuple(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _reconcile_calendar_replies(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
profile_id: str,
|
||||||
|
folder: str,
|
||||||
|
uid: str,
|
||||||
|
raw_message: bytes,
|
||||||
|
) -> int:
|
||||||
|
provider = calendar_invitation_provider(get_registry())
|
||||||
|
if provider is None:
|
||||||
|
return 0
|
||||||
|
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
|
||||||
|
try:
|
||||||
|
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||||
|
message_id = normalize_message_id(message.get("Message-ID"))
|
||||||
|
except Exception:
|
||||||
|
message_id = None
|
||||||
|
recorded_count = 0
|
||||||
|
for icalendar, filename in _calendar_reply_parts(raw_message):
|
||||||
|
try:
|
||||||
|
recorded = provider.record_icalendar_reply(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
icalendar=icalendar,
|
||||||
|
received_at=utcnow(),
|
||||||
|
evidence={
|
||||||
|
"transport": "mail-imap",
|
||||||
|
"profile_id": profile_id,
|
||||||
|
"folder": folder,
|
||||||
|
"mailbox_uid": uid,
|
||||||
|
"message_id": message_id,
|
||||||
|
"filename": filename,
|
||||||
|
"raw_sha256": raw_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except CalendarCapabilityError:
|
||||||
|
# Mailboxes routinely contain unrelated or malformed invitations.
|
||||||
|
# They are not allowed to block DSN progress for the entire source.
|
||||||
|
continue
|
||||||
|
for invitation in recorded:
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
user_id=None,
|
||||||
|
action="mail.calendar_reply.reconciled",
|
||||||
|
object_type="calendar_invitation",
|
||||||
|
object_id=invitation.event_id,
|
||||||
|
details={
|
||||||
|
"profile_id": profile_id,
|
||||||
|
"folder": folder,
|
||||||
|
"mailbox_uid": uid,
|
||||||
|
"message_id": message_id,
|
||||||
|
"calendar_uid": invitation.uid,
|
||||||
|
"correlation_id": invitation.correlation_id,
|
||||||
|
"raw_sha256": raw_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
recorded_count += 1
|
||||||
|
return recorded_count
|
||||||
|
|
||||||
|
|
||||||
def configure_bounce_source(
|
def configure_bounce_source(
|
||||||
session: Session,
|
session: Session,
|
||||||
*,
|
*,
|
||||||
@@ -154,7 +249,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
source = db.get(MailBounceSource, source_id)
|
source = db.get(MailBounceSource, source_id)
|
||||||
if source is None or source.tenant_id != tenant_id:
|
if source is None or source.tenant_id != tenant_id:
|
||||||
raise MailBounceError("Bounce source not found.")
|
raise MailBounceError("Bounce source not found.")
|
||||||
processed, observations = self._scan_source(
|
processed, observations, calendar_replies = self._scan_source(
|
||||||
db,
|
db,
|
||||||
source,
|
source,
|
||||||
limit=max(1, min(int(limit), 1_000)),
|
limit=max(1, min(int(limit), 1_000)),
|
||||||
@@ -163,6 +258,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
"sources": 1,
|
"sources": 1,
|
||||||
"processed_messages": processed,
|
"processed_messages": processed,
|
||||||
"observations": observations,
|
"observations": observations,
|
||||||
|
"calendar_replies": calendar_replies,
|
||||||
"failures": [],
|
"failures": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,8 +271,18 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
folder: str,
|
folder: str,
|
||||||
uid: str,
|
uid: str,
|
||||||
raw_message: bytes,
|
raw_message: bytes,
|
||||||
|
reconcile_calendar: bool = True,
|
||||||
) -> tuple[MailBounceObservationRef, ...]:
|
) -> tuple[MailBounceObservationRef, ...]:
|
||||||
db = _session(session)
|
db = _session(session)
|
||||||
|
if reconcile_calendar:
|
||||||
|
_reconcile_calendar_replies(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
profile_id=profile_id,
|
||||||
|
folder=folder,
|
||||||
|
uid=uid,
|
||||||
|
raw_message=raw_message,
|
||||||
|
)
|
||||||
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
|
raw_sha256 = hashlib.sha256(raw_message).hexdigest()
|
||||||
reports = parse_delivery_status(raw_message)
|
reports = parse_delivery_status(raw_message)
|
||||||
observations: list[MailBounceObservationRef] = []
|
observations: list[MailBounceObservationRef] = []
|
||||||
@@ -286,14 +392,20 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
)
|
)
|
||||||
processed = 0
|
processed = 0
|
||||||
observations = 0
|
observations = 0
|
||||||
|
calendar_replies = 0
|
||||||
failures: list[dict[str, str]] = []
|
failures: list[dict[str, str]] = []
|
||||||
for source in sources:
|
for source in sources:
|
||||||
if remaining <= 0:
|
if remaining <= 0:
|
||||||
break
|
break
|
||||||
try:
|
try:
|
||||||
count, found = self._scan_source(db, source, limit=remaining)
|
count, found, replies = self._scan_source(
|
||||||
|
db,
|
||||||
|
source,
|
||||||
|
limit=remaining,
|
||||||
|
)
|
||||||
processed += count
|
processed += count
|
||||||
observations += found
|
observations += found
|
||||||
|
calendar_replies += replies
|
||||||
remaining -= count
|
remaining -= count
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
source.last_scanned_at = utcnow()
|
source.last_scanned_at = utcnow()
|
||||||
@@ -304,6 +416,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
"sources": len(sources),
|
"sources": len(sources),
|
||||||
"processed_messages": processed,
|
"processed_messages": processed,
|
||||||
"observations": observations,
|
"observations": observations,
|
||||||
|
"calendar_replies": calendar_replies,
|
||||||
"failures": failures,
|
"failures": failures,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -336,7 +449,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
source: MailBounceSource,
|
source: MailBounceSource,
|
||||||
*,
|
*,
|
||||||
limit: int,
|
limit: int,
|
||||||
) -> tuple[int, int]:
|
) -> tuple[int, int, int]:
|
||||||
profile = _profile(
|
profile = _profile(
|
||||||
session,
|
session,
|
||||||
tenant_id=source.tenant_id,
|
tenant_id=source.tenant_id,
|
||||||
@@ -360,6 +473,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
limit=limit,
|
limit=limit,
|
||||||
)
|
)
|
||||||
found = 0
|
found = 0
|
||||||
|
calendar_replies = 0
|
||||||
highest = 0 if page.cursor_reset else source.highest_processed_uid
|
highest = 0 if page.cursor_reset else source.highest_processed_uid
|
||||||
for uid in page.uids:
|
for uid in page.uids:
|
||||||
raw = get_imap_raw_message(
|
raw = get_imap_raw_message(
|
||||||
@@ -367,6 +481,14 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
folder=source.folder,
|
folder=source.folder,
|
||||||
uid=uid,
|
uid=uid,
|
||||||
)
|
)
|
||||||
|
calendar_replies += _reconcile_calendar_replies(
|
||||||
|
session,
|
||||||
|
tenant_id=source.tenant_id,
|
||||||
|
profile_id=source.profile_id,
|
||||||
|
folder=source.folder,
|
||||||
|
uid=uid,
|
||||||
|
raw_message=raw.raw,
|
||||||
|
)
|
||||||
found += len(
|
found += len(
|
||||||
self.process_raw_message(
|
self.process_raw_message(
|
||||||
session,
|
session,
|
||||||
@@ -375,6 +497,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
folder=source.folder,
|
folder=source.folder,
|
||||||
uid=uid,
|
uid=uid,
|
||||||
raw_message=raw.raw,
|
raw_message=raw.raw,
|
||||||
|
reconcile_calendar=False,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
highest = max(highest, int(uid))
|
highest = max(highest, int(uid))
|
||||||
@@ -385,7 +508,7 @@ class SqlMailBounceProcessingProvider(MailBounceProcessingProvider):
|
|||||||
source.last_success_at = now
|
source.last_success_at = now
|
||||||
source.last_error = None
|
source.last_error = None
|
||||||
session.flush()
|
session.flush()
|
||||||
return len(page.uids), found
|
return len(page.uids), found, calendar_replies
|
||||||
|
|
||||||
|
|
||||||
def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]:
|
def parse_delivery_status(raw_message: bytes) -> tuple[Mapping[str, object], ...]:
|
||||||
|
|||||||
@@ -299,7 +299,7 @@ manifest = ModuleManifest(
|
|||||||
name="Mail",
|
name="Mail",
|
||||||
version="0.1.10",
|
version="0.1.10",
|
||||||
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
|
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
|
||||||
optional_dependencies=("campaigns", "addresses"),
|
optional_dependencies=("campaigns", "addresses", "calendar"),
|
||||||
provides_interfaces=(
|
provides_interfaces=(
|
||||||
ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.2.0"),
|
ModuleInterfaceProvider(name="mail.campaign_delivery", version="0.2.0"),
|
||||||
ModuleInterfaceProvider(name="mail.delivery_commands", version="0.1.0"),
|
ModuleInterfaceProvider(name="mail.delivery_commands", version="0.1.0"),
|
||||||
@@ -308,6 +308,12 @@ manifest = ModuleManifest(
|
|||||||
ModuleInterfaceProvider(name="mail.bounce_processing", version="0.1.0"),
|
ModuleInterfaceProvider(name="mail.bounce_processing", version="0.1.0"),
|
||||||
),
|
),
|
||||||
requires_interfaces=(
|
requires_interfaces=(
|
||||||
|
ModuleInterfaceRequirement(
|
||||||
|
name="calendar.invitations",
|
||||||
|
version_min="0.2.0",
|
||||||
|
version_max_exclusive="0.3.0",
|
||||||
|
optional=True,
|
||||||
|
),
|
||||||
ModuleInterfaceRequirement(
|
ModuleInterfaceRequirement(
|
||||||
name="campaigns.access",
|
name="campaigns.access",
|
||||||
version_min="0.1.0",
|
version_min="0.1.0",
|
||||||
@@ -444,7 +450,10 @@ manifest = ModuleManifest(
|
|||||||
"bounded message/delivery-status reports without changing mailbox flags, "
|
"bounded message/delivery-status reports without changing mailbox flags, "
|
||||||
"and records idempotent per-recipient observations. SMTP acceptance remains "
|
"and records idempotent per-recipient observations. SMTP acceptance remains "
|
||||||
"separate from a later bounce. Unmatched reports remain visible for review; "
|
"separate from a later bounce. Unmatched reports remain visible for review; "
|
||||||
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body."
|
"Mail stores only bounded diagnostics and a raw digest, not the raw bounce body. "
|
||||||
|
"When Calendar is active, the same bounded source scan forwards METHOD:REPLY "
|
||||||
|
"iCalendar parts to Calendar for idempotent attendee reconciliation without "
|
||||||
|
"classifying the message as a bounce."
|
||||||
),
|
),
|
||||||
layer="configured",
|
layer="configured",
|
||||||
documentation_types=("admin", "user"),
|
documentation_types=("admin", "user"),
|
||||||
@@ -456,7 +465,7 @@ manifest = ModuleManifest(
|
|||||||
any_scopes=("mail:bounce:read", "mail:bounce:manage"),
|
any_scopes=("mail:bounce:read", "mail:bounce:manage"),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
related_modules=("campaigns", "audit"),
|
related_modules=("campaigns", "calendar", "audit"),
|
||||||
metadata={
|
metadata={
|
||||||
"kind": "workflow",
|
"kind": "workflow",
|
||||||
"route": "/mail/bounces",
|
"route": "/mail/bounces",
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
|||||||
from sqlalchemy.pool import StaticPool
|
from sqlalchemy.pool import StaticPool
|
||||||
|
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.core.calendar import CalendarInvitationRef
|
||||||
from govoplan_mail.backend.bounce_processing import (
|
from govoplan_mail.backend.bounce_processing import (
|
||||||
SqlMailBounceProcessingProvider,
|
SqlMailBounceProcessingProvider,
|
||||||
parse_delivery_status,
|
parse_delivery_status,
|
||||||
@@ -57,6 +58,24 @@ Body
|
|||||||
--dsn--
|
--dsn--
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
CALENDAR_REPLY = b"""From: Ada <ada@example.test>
|
||||||
|
To: organizer@example.test
|
||||||
|
Message-ID: <calendar-reply-1@example.test>
|
||||||
|
Subject: Accepted: Planning
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Type: text/calendar; method=REPLY; charset=utf-8
|
||||||
|
|
||||||
|
BEGIN:VCALENDAR
|
||||||
|
VERSION:2.0
|
||||||
|
METHOD:REPLY
|
||||||
|
BEGIN:VEVENT
|
||||||
|
UID:invitation-1@govoplan.local
|
||||||
|
DTSTART:20260805T090000Z
|
||||||
|
ATTENDEE;PARTSTAT=ACCEPTED:mailto:ada@example.test
|
||||||
|
END:VEVENT
|
||||||
|
END:VCALENDAR
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
class MailBounceProcessingTests(unittest.TestCase):
|
class MailBounceProcessingTests(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
@@ -181,6 +200,49 @@ class MailBounceProcessingTests(unittest.TestCase):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_calendar_reply_is_forwarded_without_becoming_a_bounce(self) -> None:
|
||||||
|
class CalendarProvider:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def record_icalendar_reply(self, session, **kwargs):
|
||||||
|
self.calls.append((session, kwargs))
|
||||||
|
return (
|
||||||
|
CalendarInvitationRef(
|
||||||
|
event_id="event-1",
|
||||||
|
calendar_id="calendar-1",
|
||||||
|
uid="invitation-1@govoplan.local",
|
||||||
|
correlation_id="campaign:version-1:entry-1",
|
||||||
|
source_module="campaigns",
|
||||||
|
source_resource_type="campaign_version",
|
||||||
|
source_resource_id="version-1",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
calendar = CalendarProvider()
|
||||||
|
provider = SqlMailBounceProcessingProvider()
|
||||||
|
with (
|
||||||
|
self.SessionLocal() as session,
|
||||||
|
patch(
|
||||||
|
"govoplan_mail.backend.bounce_processing.calendar_invitation_provider",
|
||||||
|
return_value=calendar,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
observations = provider.process_raw_message(
|
||||||
|
session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
profile_id="profile-1",
|
||||||
|
folder="INBOX",
|
||||||
|
uid="43",
|
||||||
|
raw_message=CALENDAR_REPLY,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual((), observations)
|
||||||
|
self.assertEqual(1, len(calendar.calls))
|
||||||
|
forwarded = calendar.calls[0][1]
|
||||||
|
self.assertIn("METHOD:REPLY", forwarded["icalendar"])
|
||||||
|
self.assertEqual("43", forwarded["evidence"]["mailbox_uid"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user