From 5a3b13e26e8061e32771ea8f1b924bd798bfa9d0 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 1 Aug 2026 17:48:37 +0200 Subject: [PATCH] feat: declare governed external provider state --- AGENTS.md | 6 + src/govoplan_mail/backend/manifest.py | 138 ++++++++ src/govoplan_mail/backend/provider_state.py | 342 ++++++++++++++++++++ tests/test_provider_state.py | 136 ++++++++ 4 files changed, 622 insertions(+) create mode 100644 src/govoplan_mail/backend/provider_state.py create mode 100644 tests/test_provider_state.py diff --git a/AGENTS.md b/AGENTS.md index c3f2102..0ef06b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,11 @@ # GovOPlaN Mail Codex Guide +## Documentation Contract + +- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior. +- Keep feature content here; `govoplan-docs` projects it without importing Mail internals. +- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes. + ## Scope This repository owns the `mail` module: SMTP/IMAP profiles, mail profile policy, encrypted mail credentials, SMTP sending, IMAP append and mailbox access, mock mail infrastructure, backend module manifest, and `@govoplan/mail-webui`. diff --git a/src/govoplan_mail/backend/manifest.py b/src/govoplan_mail/backend/manifest.py index 8433276..f016272 100644 --- a/src/govoplan_mail/backend/manifest.py +++ b/src/govoplan_mail/backend/manifest.py @@ -28,12 +28,25 @@ from govoplan_core.core.modules import ( PermissionDefinition, RoleTemplate, ) +from govoplan_core.core.provider_governance import ( + ExternalProviderDeclaration, + ExternalProviderStateProviderRegistration, + ProviderBehaviorDeclaration, + ProviderObjectDeclaration, + declared_module_architecture, +) from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base from govoplan_mail.backend.documentation import ( documentation_configuration_states, documentation_topics, ) +from govoplan_mail.backend.provider_state import ( + IMAP_PROVIDER_ID, + SMTP_PROVIDER_ID, + imap_provider_states, + smtp_provider_states, +) from govoplan_mail.backend.db import models as mail_models # noqa: F401 - populate Mail ORM metadata @@ -188,6 +201,99 @@ def _mail_router(context: ModuleContext): return aggregate +SMTP_PROVIDER = ExternalProviderDeclaration( + id=SMTP_PROVIDER_ID, + module_id="mail", + label="SMTP message delivery", + maturity="publish", + operations=("discover", "read", "publish", "preview", "dry_run"), + objects=( + ProviderObjectDeclaration( + object_type="mail_server_endpoint", + field_groups=("identity", "transport", "policy", "revision"), + authority_modes=("external_authoritative", "governance_overlay"), + default_authority_mode="governance_overlay", + ), + ProviderObjectDeclaration( + object_type="outbound_message", + field_groups=("envelope", "content_digest", "delivery_state", "evidence"), + authority_modes=("governance_overlay",), + default_authority_mode="governance_overlay", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="Every command pins the selected SMTP endpoint, credential, and random transport revision.", + concurrency="The expected transport revision is checked before credentials are decrypted or an effect starts.", + freshness="Delivery outcomes and completion times are retained; transport freshness is not periodic state.", + health="Successful deliveries and unresolved outcome-unknown commands are projected without exposing server details.", + max_read_items=5000, + idempotency="Tenant, command type, and idempotency key bind one canonical encrypted delivery request.", + retry="Only classified pre-acceptance temporary failures are retried with bounded scheduling.", + timeout_seconds=60, + conflicts="A reused idempotency key with different content is rejected before delivery.", + outcome_unknown="A connection failure after effect start becomes outcome_unknown and is never blindly retried.", + outcome_unknown_supported=True, + evidence="Encrypted command, attempt, acceptance/refusal summary, effect-start marker, and reconciliation are retained.", + audit_event_types=( + "mail.delivery_requested", + "mail.delivery_completed", + "mail.delivery_reconciled", + ), + correction="A reconciled new command is a separate auditable effect and does not rewrite the original outcome.", + rollback="SMTP acceptance cannot be rolled back.", + compensation="A follow-up message or domain correction is the only safe compensation after acceptance.", + reconciliation="Operators record provider evidence and choose accepted or not-accepted before any resend.", + outage="Pending commands remain durable; accepted or outcome-unknown commands are not redelivered automatically.", + classifications=("confidential", "personal", "special_category"), + purposes=("governed message delivery", "notification delivery"), + retention="Payload, delivery evidence, and audit records follow separate configured retention policies.", + secret_handling="Credentials are decrypted only inside Mail after authorization and revision validation.", + ), + interface_names=("mail.campaign_delivery", "mail.delivery_commands", "mail.delivery_outbox"), + documentation_topic_ids=("mail.reference.campaign-delivery-contract",), +) + + +IMAP_PROVIDER = ExternalProviderDeclaration( + id=IMAP_PROVIDER_ID, + module_id="mail", + label="IMAP mailbox projection", + maturity="read", + operations=("discover", "search", "read", "preview"), + objects=( + ProviderObjectDeclaration( + object_type="mailbox_folder", + field_groups=("identity", "flags", "counts", "revision"), + authority_modes=("external_authoritative", "external_mirror"), + default_authority_mode="external_mirror", + ), + ProviderObjectDeclaration( + object_type="mailbox_message", + field_groups=("identity", "headers", "body_preview", "flags", "source_metadata"), + authority_modes=("external_authoritative", "external_mirror"), + default_authority_mode="external_mirror", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="UIDVALIDITY, UID, folder, flags, and transport revision identify mailbox observations.", + concurrency="Bounded reads pin the profile transport revision and never mutate message flags.", + freshness="Folder and message index timestamps state when the external mailbox was last observed.", + health="Index state and bounce-source errors are projected independently of secret profile fields.", + max_read_items=500, + evidence="Mailbox index rows and bounce observations retain bounded source identities and observation times.", + audit_event_types=("mail.mailbox.read", "mail.bounce.observed"), + correction="A later mailbox refresh replaces the derived projection while source-owned history remains external.", + reconciliation="UIDVALIDITY changes invalidate the affected derived index before a bounded refresh.", + outage="The last derived index remains readable with stale or unknown freshness where policy permits.", + classifications=("confidential", "personal", "special_category"), + purposes=("mailbox access", "delivery-status processing"), + retention="Derived mailbox indexes and bounce evidence follow Mail retention policy.", + secret_handling="IMAP credentials remain encrypted and are never returned through mailbox or provider-state APIs.", + ), + documentation_topic_ids=("mail.workflow.read-mailbox",), +) + + manifest = ModuleManifest( id="mail", name="Mail", @@ -563,6 +669,38 @@ manifest = ModuleManifest( resolve=documentation_configuration_states, ), ), + external_providers=(SMTP_PROVIDER, IMAP_PROVIDER), + external_provider_state_providers=( + ExternalProviderStateProviderRegistration( + module_id="mail", + provider_id=SMTP_PROVIDER_ID, + provider=smtp_provider_states, + ), + ExternalProviderStateProviderRegistration( + module_id="mail", + provider_id=IMAP_PROVIDER_ID, + provider=imap_provider_states, + ), + ), + architecture=declared_module_architecture( + layer="communication_participation", + kind="integration", + maturity="vertical_slice", + documentation_ref="docs/MAIL_HANDBOOK.md", + test_ref="tests/test_delivery_outbox.py", + known_limits=("Provider recovery drills and a complete webmail profile are not reference-ready.",), + supported_authority_modes=( + "external_authoritative", + "external_mirror", + "governance_overlay", + ), + owned_concepts=("mail profile", "mail delivery command", "delivery attempt", "mailbox projection"), + non_owned_concepts=("campaign", "notification", "recipient address directory", "external mailbox"), + target_tested_providers=(SMTP_PROVIDER_ID, IMAP_PROVIDER_ID), + recovery_docs=("docs/MAIL_HANDBOOK.md",), + security_docs=("docs/MAIL_HANDBOOK.md",), + operations_docs=("docs/MAIL_HANDBOOK.md",), + ), ) diff --git a/src/govoplan_mail/backend/provider_state.py b/src/govoplan_mail/backend/provider_state.py new file mode 100644 index 0000000..ff444ab --- /dev/null +++ b/src/govoplan_mail/backend/provider_state.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +from collections import defaultdict +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import case, func, or_, select +from sqlalchemy.orm import Session + +from govoplan_core.core.provider_governance import ( + ExternalProviderRuntimeState, + ExternalProviderStateContext, +) +from govoplan_mail.backend.db.models import ( + MailBounceSource, + MailDeliveryCommand, + MailMailboxFolderIndex, + MailMailboxMessageIndex, + MailServerEndpoint, + MailServerProfile, +) + + +SMTP_PROVIDER_ID = "mail.smtp_delivery" +IMAP_PROVIDER_ID = "mail.imap_mailbox" +_CURRENT_INDEX_WINDOW = timedelta(minutes=30) + + +def smtp_provider_states( + context: ExternalProviderStateContext, +) -> tuple[ExternalProviderRuntimeState, ...]: + return _mail_provider_states(context, protocol="smtp") + + +def imap_provider_states( + context: ExternalProviderStateContext, +) -> tuple[ExternalProviderRuntimeState, ...]: + return _mail_provider_states(context, protocol="imap") + + +def _mail_provider_states( + context: ExternalProviderStateContext, + *, + protocol: str, +) -> tuple[ExternalProviderRuntimeState, ...]: + if not isinstance(context.session, Session): + raise RuntimeError("Mail provider state requires a database session.") + profiles = _profiles(context) + if not profiles: + return () + profile_ids = tuple(item.id for item in profiles) + endpoints = _endpoints(context.session, profile_ids=profile_ids, protocol=protocol) + endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list) + for endpoint in endpoints: + endpoints_by_profile[endpoint.profile_id].append(endpoint) + + observed_at = datetime.now(UTC) + if protocol == "smtp": + metrics = _smtp_metrics(context.session, profile_ids=profile_ids) + return tuple( + _smtp_state( + profile, + endpoints=endpoints_by_profile.get(profile.id, []), + metrics=metrics.get(profile.id, {}), + observed_at=observed_at, + ) + for profile in profiles + if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "smtp") + ) + + metrics = _imap_metrics(context.session, profile_ids=profile_ids) + return tuple( + _imap_state( + profile, + endpoints=endpoints_by_profile.get(profile.id, []), + metrics=metrics.get(profile.id, {}), + observed_at=observed_at, + ) + for profile in profiles + if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "imap") + ) + + +def _profiles(context: ExternalProviderStateContext) -> tuple[MailServerProfile, ...]: + statement = select(MailServerProfile) + if context.tenant_id is not None: + statement = statement.where( + or_( + MailServerProfile.tenant_id.is_(None), + MailServerProfile.tenant_id == context.tenant_id, + ) + ) + return tuple( + context.session.scalars( + statement.order_by( + MailServerProfile.tenant_id, + MailServerProfile.id, + ).limit(context.max_items + 1) + ) + ) + + +def _endpoints( + session: Session, + *, + profile_ids: tuple[str, ...], + protocol: str, +) -> tuple[MailServerEndpoint, ...]: + return tuple( + session.scalars( + select(MailServerEndpoint).where( + MailServerEndpoint.profile_id.in_(profile_ids), + MailServerEndpoint.protocol == protocol, + ) + ) + ) + + +def _smtp_metrics( + session: Session, + *, + profile_ids: tuple[str, ...], +) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = defaultdict(dict) + rows = session.execute( + select( + MailDeliveryCommand.profile_id, + MailDeliveryCommand.status, + func.count(MailDeliveryCommand.id), + func.max(MailDeliveryCommand.completed_at), + ) + .where(MailDeliveryCommand.profile_id.in_(profile_ids)) + .group_by(MailDeliveryCommand.profile_id, MailDeliveryCommand.status) + ) + for profile_id, status, count, last_completed_at in rows: + item = result[str(profile_id)] + item[str(status)] = int(count) + if status in {"accepted", "reconciled_accepted", "partially_refused"}: + current = _aware(item.get("last_success_at")) + candidate = _aware(last_completed_at) + if candidate is not None and (current is None or candidate > current): + item["last_success_at"] = candidate + return result + + +def _imap_metrics( + session: Session, + *, + profile_ids: tuple[str, ...], +) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = defaultdict(dict) + folder_rows = session.execute( + select( + MailMailboxFolderIndex.profile_id, + func.count(MailMailboxFolderIndex.id), + func.max(MailMailboxFolderIndex.indexed_at), + ) + .where(MailMailboxFolderIndex.profile_id.in_(profile_ids)) + .group_by(MailMailboxFolderIndex.profile_id) + ) + for profile_id, count, indexed_at in folder_rows: + result[str(profile_id)]["indexed_folders"] = int(count) + result[str(profile_id)]["last_indexed_at"] = _aware(indexed_at) + message_rows = session.execute( + select( + MailMailboxMessageIndex.profile_id, + func.count(MailMailboxMessageIndex.id), + ) + .where(MailMailboxMessageIndex.profile_id.in_(profile_ids)) + .group_by(MailMailboxMessageIndex.profile_id) + ) + for profile_id, count in message_rows: + result[str(profile_id)]["indexed_messages"] = int(count) + bounce_rows = session.execute( + select( + MailBounceSource.profile_id, + func.count(MailBounceSource.id), + func.sum( + case((MailBounceSource.last_error.is_not(None), 1), else_=0) + ), + func.max(MailBounceSource.last_success_at), + ) + .where( + MailBounceSource.profile_id.in_(profile_ids), + MailBounceSource.is_active.is_(True), + ) + .group_by(MailBounceSource.profile_id) + ) + for profile_id, count, errors, last_success_at in bounce_rows: + item = result[str(profile_id)] + item["active_bounce_sources"] = int(count) + item["bounce_source_errors"] = int(errors or 0) + item["last_bounce_success_at"] = _aware(last_success_at) + return result + + +def _smtp_state( + profile: MailServerProfile, + *, + endpoints: list[MailServerEndpoint], + metrics: dict[str, Any], + observed_at: datetime, +) -> ExternalProviderRuntimeState: + active = bool(profile.is_active) and ( + any(item.is_active for item in endpoints) + or (not endpoints and _legacy_configured(profile, "smtp")) + ) + outcome_unknown = int(metrics.get("outcome_unknown", 0)) + last_success = _aware(metrics.get("last_success_at")) + health = ( + "inactive" + if not active + else "warning" + if outcome_unknown + else "healthy" + if last_success is not None + else "unknown" + ) + return ExternalProviderRuntimeState( + provider_id=SMTP_PROVIDER_ID, + binding_ref=f"mail:profile:{profile.id}:smtp", + authority_mode="governance_overlay", + observed_at=observed_at, + configured=True, + active=active, + health=health, + freshness="not_applicable", + conflict="pending" if outcome_unknown else "clear", + recovery=( + "not_applicable" + if not active + else "attention" + if outcome_unknown or last_success is None + else "ready" + ), + last_success_at=last_success, + detail=( + "SMTP delivery is disabled." + if not active + else "SMTP outcomes require reconciliation." + if outcome_unknown + else "SMTP delivery has no retained successful observation yet." + if last_success is None + else "SMTP delivery evidence is available." + ), + metrics={ + "pending_commands": int(metrics.get("pending", 0)), + "temporary_failures": int(metrics.get("temporary_failure", 0)), + "permanent_failures": int(metrics.get("permanent_failure", 0)), + "outcome_unknown_commands": outcome_unknown, + "active_endpoints": sum(1 for item in endpoints if item.is_active), + }, + ) + + +def _imap_state( + profile: MailServerProfile, + *, + endpoints: list[MailServerEndpoint], + metrics: dict[str, Any], + observed_at: datetime, +) -> ExternalProviderRuntimeState: + active = bool(profile.is_active) and ( + any(item.is_active for item in endpoints) + or (not endpoints and _legacy_configured(profile, "imap")) + ) + indexed_at = _aware(metrics.get("last_indexed_at")) + errors = int(metrics.get("bounce_source_errors", 0)) + freshness = ( + "not_applicable" + if not active + else "unknown" + if indexed_at is None + else "current" + if observed_at - indexed_at <= _CURRENT_INDEX_WINDOW + else "stale" + ) + health = ( + "inactive" + if not active + else "error" + if errors + else "healthy" + if indexed_at is not None + else "unknown" + ) + return ExternalProviderRuntimeState( + provider_id=IMAP_PROVIDER_ID, + binding_ref=f"mail:profile:{profile.id}:imap", + authority_mode="external_mirror", + observed_at=observed_at, + configured=True, + active=active, + health=health, + freshness=freshness, + conflict="not_applicable", + recovery=( + "not_applicable" + if not active + else "ready" + if health == "healthy" and freshness == "current" + else "attention" + ), + last_success_at=indexed_at or _aware(metrics.get("last_bounce_success_at")), + detail=( + "IMAP mailbox access is disabled." + if not active + else "IMAP mailbox or bounce-source errors require attention." + if errors + else "IMAP mailbox state has not been indexed yet." + if indexed_at is None + else "IMAP mailbox index state is available." + ), + metrics={ + "indexed_folders": int(metrics.get("indexed_folders", 0)), + "indexed_messages": int(metrics.get("indexed_messages", 0)), + "active_bounce_sources": int(metrics.get("active_bounce_sources", 0)), + "bounce_source_errors": errors, + "active_endpoints": sum(1 for item in endpoints if item.is_active), + }, + ) + + +def _legacy_configured(profile: MailServerProfile, protocol: str) -> bool: + value = profile.smtp_config if protocol == "smtp" else profile.imap_config + return isinstance(value, dict) and bool(value) + + +def _aware(value: object | None) -> datetime | None: + if not isinstance(value, datetime): + return None + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +__all__ = [ + "IMAP_PROVIDER_ID", + "SMTP_PROVIDER_ID", + "imap_provider_states", + "smtp_provider_states", +] diff --git a/tests/test_provider_state.py b/tests/test_provider_state.py new file mode 100644 index 0000000..9b13809 --- /dev/null +++ b/tests/test_provider_state.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_access.backend.db import models as access_models # noqa: F401 +from govoplan_core.core.provider_governance import ExternalProviderStateContext +from govoplan_core.db.base import Base +from govoplan_mail.backend.db.models import ( + MailBounceSource, + MailDeliveryCommand, + MailMailboxFolderIndex, + MailMailboxMessageIndex, + MailServerEndpoint, + MailServerProfile, +) +from govoplan_mail.backend.manifest import manifest +from govoplan_mail.backend.provider_state import ( + IMAP_PROVIDER_ID, + SMTP_PROVIDER_ID, + imap_provider_states, + smtp_provider_states, +) + + +class MailProviderStateTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True) + Base.metadata.create_all( + self.engine, + tables=( + MailServerProfile.__table__, + MailServerEndpoint.__table__, + MailDeliveryCommand.__table__, + MailMailboxFolderIndex.__table__, + MailMailboxMessageIndex.__table__, + MailBounceSource.__table__, + ), + ) + self.session = sessionmaker(bind=self.engine, expire_on_commit=False)() + self.profile = MailServerProfile( + id="profile-1", + tenant_id="tenant-1", + scope_type="tenant", + scope_id="tenant-1", + name="Mail", + slug="mail", + is_active=True, + smtp_config={"host": "smtp.example.test", "port": 587}, + imap_config={"host": "imap.example.test", "port": 993}, + ) + self.session.add( + self.profile + ) + self.session.add( + MailMailboxFolderIndex( + id="folder-1", + tenant_id="tenant-1", + profile_id=self.profile.id, + folder="INBOX", + indexed_at=datetime.now(UTC), + ) + ) + self.session.commit() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_smtp_state_exposes_reconciliation_without_server_or_secrets(self) -> None: + self.session.add( + MailDeliveryCommand( + id="command-1", + tenant_id="tenant-1", + command_type="message", + source_module="notifications", + source_resource_type="notification", + idempotency_key="notification-1", + canonical_request_hash="a" * 64, + profile_id=self.profile.id, + expected_smtp_transport_revision="revision-1", + message_sha256="b" * 64, + message_size_bytes=10, + recipient_count=1, + status="outcome_unknown", + expires_at=datetime.now(UTC) + timedelta(days=1), + ) + ) + self.session.flush() + + state = smtp_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + + self.assertEqual("warning", state.health) + self.assertEqual("pending", state.conflict) + self.assertEqual("attention", state.recovery) + self.assertEqual(1, state.metrics["outcome_unknown_commands"]) + self.assertNotIn("smtp.example.test", str(state.to_dict())) + + def test_imap_state_is_fresh_tenant_bounded_and_registered(self) -> None: + state = imap_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + + self.assertEqual("healthy", state.health) + self.assertEqual("current", state.freshness) + self.assertEqual("external_mirror", state.authority_mode) + self.assertNotIn("imap.example.test", str(state.to_dict())) + self.assertEqual( + (), + imap_provider_states( + ExternalProviderStateContext( + session=self.session, + tenant_id="tenant-2", + ) + ), + ) + self.assertEqual( + {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID}, + {item.id for item in manifest.external_providers}, + ) + self.assertEqual( + {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID}, + { + item.provider_id + for item in manifest.external_provider_state_providers + }, + ) + + +if __name__ == "__main__": + unittest.main()