feat: declare governed external provider state

AGENTS.md

@@ -1,5 +1,11 @@
 # GovOPlaN Mail Codex Guide
 
+## Documentation Contract
+
+- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
+- Keep feature content here; `govoplan-docs` projects it without importing Mail internals.
+- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
+
 ## Scope
 
 This repository owns the `mail` module: SMTP/IMAP profiles, mail profile policy, encrypted mail credentials, SMTP sending, IMAP append and mailbox access, mock mail infrastructure, backend module manifest, and `@govoplan/mail-webui`.

src/govoplan_mail/backend/manifest.py

@@ -28,12 +28,25 @@ from govoplan_core.core.modules import (
     PermissionDefinition,
     RoleTemplate,
 )
+from govoplan_core.core.provider_governance import (
+    ExternalProviderDeclaration,
+    ExternalProviderStateProviderRegistration,
+    ProviderBehaviorDeclaration,
+    ProviderObjectDeclaration,
+    declared_module_architecture,
+)
 from govoplan_core.core.views import ViewSurface
 from govoplan_core.db.base import Base
 from govoplan_mail.backend.documentation import (
     documentation_configuration_states,
     documentation_topics,
 )
+from govoplan_mail.backend.provider_state import (
+    IMAP_PROVIDER_ID,
+    SMTP_PROVIDER_ID,
+    imap_provider_states,
+    smtp_provider_states,
+)
 from govoplan_mail.backend.db import models as mail_models  # noqa: F401 - populate Mail ORM metadata
 
 
@@ -188,6 +201,99 @@ def _mail_router(context: ModuleContext):
     return aggregate
 
 
+SMTP_PROVIDER = ExternalProviderDeclaration(
+    id=SMTP_PROVIDER_ID,
+    module_id="mail",
+    label="SMTP message delivery",
+    maturity="publish",
+    operations=("discover", "read", "publish", "preview", "dry_run"),
+    objects=(
+        ProviderObjectDeclaration(
+            object_type="mail_server_endpoint",
+            field_groups=("identity", "transport", "policy", "revision"),
+            authority_modes=("external_authoritative", "governance_overlay"),
+            default_authority_mode="governance_overlay",
+        ),
+        ProviderObjectDeclaration(
+            object_type="outbound_message",
+            field_groups=("envelope", "content_digest", "delivery_state", "evidence"),
+            authority_modes=("governance_overlay",),
+            default_authority_mode="governance_overlay",
+        ),
+    ),
+    behavior=ProviderBehaviorDeclaration(
+        revision_tokens="Every command pins the selected SMTP endpoint, credential, and random transport revision.",
+        concurrency="The expected transport revision is checked before credentials are decrypted or an effect starts.",
+        freshness="Delivery outcomes and completion times are retained; transport freshness is not periodic state.",
+        health="Successful deliveries and unresolved outcome-unknown commands are projected without exposing server details.",
+        max_read_items=5000,
+        idempotency="Tenant, command type, and idempotency key bind one canonical encrypted delivery request.",
+        retry="Only classified pre-acceptance temporary failures are retried with bounded scheduling.",
+        timeout_seconds=60,
+        conflicts="A reused idempotency key with different content is rejected before delivery.",
+        outcome_unknown="A connection failure after effect start becomes outcome_unknown and is never blindly retried.",
+        outcome_unknown_supported=True,
+        evidence="Encrypted command, attempt, acceptance/refusal summary, effect-start marker, and reconciliation are retained.",
+        audit_event_types=(
+            "mail.delivery_requested",
+            "mail.delivery_completed",
+            "mail.delivery_reconciled",
+        ),
+        correction="A reconciled new command is a separate auditable effect and does not rewrite the original outcome.",
+        rollback="SMTP acceptance cannot be rolled back.",
+        compensation="A follow-up message or domain correction is the only safe compensation after acceptance.",
+        reconciliation="Operators record provider evidence and choose accepted or not-accepted before any resend.",
+        outage="Pending commands remain durable; accepted or outcome-unknown commands are not redelivered automatically.",
+        classifications=("confidential", "personal", "special_category"),
+        purposes=("governed message delivery", "notification delivery"),
+        retention="Payload, delivery evidence, and audit records follow separate configured retention policies.",
+        secret_handling="Credentials are decrypted only inside Mail after authorization and revision validation.",
+    ),
+    interface_names=("mail.campaign_delivery", "mail.delivery_commands", "mail.delivery_outbox"),
+    documentation_topic_ids=("mail.reference.campaign-delivery-contract",),
+)
+
+
+IMAP_PROVIDER = ExternalProviderDeclaration(
+    id=IMAP_PROVIDER_ID,
+    module_id="mail",
+    label="IMAP mailbox projection",
+    maturity="read",
+    operations=("discover", "search", "read", "preview"),
+    objects=(
+        ProviderObjectDeclaration(
+            object_type="mailbox_folder",
+            field_groups=("identity", "flags", "counts", "revision"),
+            authority_modes=("external_authoritative", "external_mirror"),
+            default_authority_mode="external_mirror",
+        ),
+        ProviderObjectDeclaration(
+            object_type="mailbox_message",
+            field_groups=("identity", "headers", "body_preview", "flags", "source_metadata"),
+            authority_modes=("external_authoritative", "external_mirror"),
+            default_authority_mode="external_mirror",
+        ),
+    ),
+    behavior=ProviderBehaviorDeclaration(
+        revision_tokens="UIDVALIDITY, UID, folder, flags, and transport revision identify mailbox observations.",
+        concurrency="Bounded reads pin the profile transport revision and never mutate message flags.",
+        freshness="Folder and message index timestamps state when the external mailbox was last observed.",
+        health="Index state and bounce-source errors are projected independently of secret profile fields.",
+        max_read_items=500,
+        evidence="Mailbox index rows and bounce observations retain bounded source identities and observation times.",
+        audit_event_types=("mail.mailbox.read", "mail.bounce.observed"),
+        correction="A later mailbox refresh replaces the derived projection while source-owned history remains external.",
+        reconciliation="UIDVALIDITY changes invalidate the affected derived index before a bounded refresh.",
+        outage="The last derived index remains readable with stale or unknown freshness where policy permits.",
+        classifications=("confidential", "personal", "special_category"),
+        purposes=("mailbox access", "delivery-status processing"),
+        retention="Derived mailbox indexes and bounce evidence follow Mail retention policy.",
+        secret_handling="IMAP credentials remain encrypted and are never returned through mailbox or provider-state APIs.",
+    ),
+    documentation_topic_ids=("mail.workflow.read-mailbox",),
+)
+
+
 manifest = ModuleManifest(
     id="mail",
     name="Mail",
@@ -563,6 +669,38 @@ manifest = ModuleManifest(
             resolve=documentation_configuration_states,
         ),
     ),
+    external_providers=(SMTP_PROVIDER, IMAP_PROVIDER),
+    external_provider_state_providers=(
+        ExternalProviderStateProviderRegistration(
+            module_id="mail",
+            provider_id=SMTP_PROVIDER_ID,
+            provider=smtp_provider_states,
+        ),
+        ExternalProviderStateProviderRegistration(
+            module_id="mail",
+            provider_id=IMAP_PROVIDER_ID,
+            provider=imap_provider_states,
+        ),
+    ),
+    architecture=declared_module_architecture(
+        layer="communication_participation",
+        kind="integration",
+        maturity="vertical_slice",
+        documentation_ref="docs/MAIL_HANDBOOK.md",
+        test_ref="tests/test_delivery_outbox.py",
+        known_limits=("Provider recovery drills and a complete webmail profile are not reference-ready.",),
+        supported_authority_modes=(
+            "external_authoritative",
+            "external_mirror",
+            "governance_overlay",
+        ),
+        owned_concepts=("mail profile", "mail delivery command", "delivery attempt", "mailbox projection"),
+        non_owned_concepts=("campaign", "notification", "recipient address directory", "external mailbox"),
+        target_tested_providers=(SMTP_PROVIDER_ID, IMAP_PROVIDER_ID),
+        recovery_docs=("docs/MAIL_HANDBOOK.md",),
+        security_docs=("docs/MAIL_HANDBOOK.md",),
+        operations_docs=("docs/MAIL_HANDBOOK.md",),
+    ),
 )
 
 

src/govoplan_mail/backend/provider_state.py

@@ -0,0 +1,342 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import case, func, or_, select
+from sqlalchemy.orm import Session
+
+from govoplan_core.core.provider_governance import (
+    ExternalProviderRuntimeState,
+    ExternalProviderStateContext,
+)
+from govoplan_mail.backend.db.models import (
+    MailBounceSource,
+    MailDeliveryCommand,
+    MailMailboxFolderIndex,
+    MailMailboxMessageIndex,
+    MailServerEndpoint,
+    MailServerProfile,
+)
+
+
+SMTP_PROVIDER_ID = "mail.smtp_delivery"
+IMAP_PROVIDER_ID = "mail.imap_mailbox"
+_CURRENT_INDEX_WINDOW = timedelta(minutes=30)
+
+
+def smtp_provider_states(
+    context: ExternalProviderStateContext,
+) -> tuple[ExternalProviderRuntimeState, ...]:
+    return _mail_provider_states(context, protocol="smtp")
+
+
+def imap_provider_states(
+    context: ExternalProviderStateContext,
+) -> tuple[ExternalProviderRuntimeState, ...]:
+    return _mail_provider_states(context, protocol="imap")
+
+
+def _mail_provider_states(
+    context: ExternalProviderStateContext,
+    *,
+    protocol: str,
+) -> tuple[ExternalProviderRuntimeState, ...]:
+    if not isinstance(context.session, Session):
+        raise RuntimeError("Mail provider state requires a database session.")
+    profiles = _profiles(context)
+    if not profiles:
+        return ()
+    profile_ids = tuple(item.id for item in profiles)
+    endpoints = _endpoints(context.session, profile_ids=profile_ids, protocol=protocol)
+    endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list)
+    for endpoint in endpoints:
+        endpoints_by_profile[endpoint.profile_id].append(endpoint)
+
+    observed_at = datetime.now(UTC)
+    if protocol == "smtp":
+        metrics = _smtp_metrics(context.session, profile_ids=profile_ids)
+        return tuple(
+            _smtp_state(
+                profile,
+                endpoints=endpoints_by_profile.get(profile.id, []),
+                metrics=metrics.get(profile.id, {}),
+                observed_at=observed_at,
+            )
+            for profile in profiles
+            if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "smtp")
+        )
+
+    metrics = _imap_metrics(context.session, profile_ids=profile_ids)
+    return tuple(
+        _imap_state(
+            profile,
+            endpoints=endpoints_by_profile.get(profile.id, []),
+            metrics=metrics.get(profile.id, {}),
+            observed_at=observed_at,
+        )
+        for profile in profiles
+        if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "imap")
+    )
+
+
+def _profiles(context: ExternalProviderStateContext) -> tuple[MailServerProfile, ...]:
+    statement = select(MailServerProfile)
+    if context.tenant_id is not None:
+        statement = statement.where(
+            or_(
+                MailServerProfile.tenant_id.is_(None),
+                MailServerProfile.tenant_id == context.tenant_id,
+            )
+        )
+    return tuple(
+        context.session.scalars(
+            statement.order_by(
+                MailServerProfile.tenant_id,
+                MailServerProfile.id,
+            ).limit(context.max_items + 1)
+        )
+    )
+
+
+def _endpoints(
+    session: Session,
+    *,
+    profile_ids: tuple[str, ...],
+    protocol: str,
+) -> tuple[MailServerEndpoint, ...]:
+    return tuple(
+        session.scalars(
+            select(MailServerEndpoint).where(
+                MailServerEndpoint.profile_id.in_(profile_ids),
+                MailServerEndpoint.protocol == protocol,
+            )
+        )
+    )
+
+
+def _smtp_metrics(
+    session: Session,
+    *,
+    profile_ids: tuple[str, ...],
+) -> dict[str, dict[str, Any]]:
+    result: dict[str, dict[str, Any]] = defaultdict(dict)
+    rows = session.execute(
+        select(
+            MailDeliveryCommand.profile_id,
+            MailDeliveryCommand.status,
+            func.count(MailDeliveryCommand.id),
+            func.max(MailDeliveryCommand.completed_at),
+        )
+        .where(MailDeliveryCommand.profile_id.in_(profile_ids))
+        .group_by(MailDeliveryCommand.profile_id, MailDeliveryCommand.status)
+    )
+    for profile_id, status, count, last_completed_at in rows:
+        item = result[str(profile_id)]
+        item[str(status)] = int(count)
+        if status in {"accepted", "reconciled_accepted", "partially_refused"}:
+            current = _aware(item.get("last_success_at"))
+            candidate = _aware(last_completed_at)
+            if candidate is not None and (current is None or candidate > current):
+                item["last_success_at"] = candidate
+    return result
+
+
+def _imap_metrics(
+    session: Session,
+    *,
+    profile_ids: tuple[str, ...],
+) -> dict[str, dict[str, Any]]:
+    result: dict[str, dict[str, Any]] = defaultdict(dict)
+    folder_rows = session.execute(
+        select(
+            MailMailboxFolderIndex.profile_id,
+            func.count(MailMailboxFolderIndex.id),
+            func.max(MailMailboxFolderIndex.indexed_at),
+        )
+        .where(MailMailboxFolderIndex.profile_id.in_(profile_ids))
+        .group_by(MailMailboxFolderIndex.profile_id)
+    )
+    for profile_id, count, indexed_at in folder_rows:
+        result[str(profile_id)]["indexed_folders"] = int(count)
+        result[str(profile_id)]["last_indexed_at"] = _aware(indexed_at)
+    message_rows = session.execute(
+        select(
+            MailMailboxMessageIndex.profile_id,
+            func.count(MailMailboxMessageIndex.id),
+        )
+        .where(MailMailboxMessageIndex.profile_id.in_(profile_ids))
+        .group_by(MailMailboxMessageIndex.profile_id)
+    )
+    for profile_id, count in message_rows:
+        result[str(profile_id)]["indexed_messages"] = int(count)
+    bounce_rows = session.execute(
+        select(
+            MailBounceSource.profile_id,
+            func.count(MailBounceSource.id),
+            func.sum(
+                case((MailBounceSource.last_error.is_not(None), 1), else_=0)
+            ),
+            func.max(MailBounceSource.last_success_at),
+        )
+        .where(
+            MailBounceSource.profile_id.in_(profile_ids),
+            MailBounceSource.is_active.is_(True),
+        )
+        .group_by(MailBounceSource.profile_id)
+    )
+    for profile_id, count, errors, last_success_at in bounce_rows:
+        item = result[str(profile_id)]
+        item["active_bounce_sources"] = int(count)
+        item["bounce_source_errors"] = int(errors or 0)
+        item["last_bounce_success_at"] = _aware(last_success_at)
+    return result
+
+
+def _smtp_state(
+    profile: MailServerProfile,
+    *,
+    endpoints: list[MailServerEndpoint],
+    metrics: dict[str, Any],
+    observed_at: datetime,
+) -> ExternalProviderRuntimeState:
+    active = bool(profile.is_active) and (
+        any(item.is_active for item in endpoints)
+        or (not endpoints and _legacy_configured(profile, "smtp"))
+    )
+    outcome_unknown = int(metrics.get("outcome_unknown", 0))
+    last_success = _aware(metrics.get("last_success_at"))
+    health = (
+        "inactive"
+        if not active
+        else "warning"
+        if outcome_unknown
+        else "healthy"
+        if last_success is not None
+        else "unknown"
+    )
+    return ExternalProviderRuntimeState(
+        provider_id=SMTP_PROVIDER_ID,
+        binding_ref=f"mail:profile:{profile.id}:smtp",
+        authority_mode="governance_overlay",
+        observed_at=observed_at,
+        configured=True,
+        active=active,
+        health=health,
+        freshness="not_applicable",
+        conflict="pending" if outcome_unknown else "clear",
+        recovery=(
+            "not_applicable"
+            if not active
+            else "attention"
+            if outcome_unknown or last_success is None
+            else "ready"
+        ),
+        last_success_at=last_success,
+        detail=(
+            "SMTP delivery is disabled."
+            if not active
+            else "SMTP outcomes require reconciliation."
+            if outcome_unknown
+            else "SMTP delivery has no retained successful observation yet."
+            if last_success is None
+            else "SMTP delivery evidence is available."
+        ),
+        metrics={
+            "pending_commands": int(metrics.get("pending", 0)),
+            "temporary_failures": int(metrics.get("temporary_failure", 0)),
+            "permanent_failures": int(metrics.get("permanent_failure", 0)),
+            "outcome_unknown_commands": outcome_unknown,
+            "active_endpoints": sum(1 for item in endpoints if item.is_active),
+        },
+    )
+
+
+def _imap_state(
+    profile: MailServerProfile,
+    *,
+    endpoints: list[MailServerEndpoint],
+    metrics: dict[str, Any],
+    observed_at: datetime,
+) -> ExternalProviderRuntimeState:
+    active = bool(profile.is_active) and (
+        any(item.is_active for item in endpoints)
+        or (not endpoints and _legacy_configured(profile, "imap"))
+    )
+    indexed_at = _aware(metrics.get("last_indexed_at"))
+    errors = int(metrics.get("bounce_source_errors", 0))
+    freshness = (
+        "not_applicable"
+        if not active
+        else "unknown"
+        if indexed_at is None
+        else "current"
+        if observed_at - indexed_at <= _CURRENT_INDEX_WINDOW
+        else "stale"
+    )
+    health = (
+        "inactive"
+        if not active
+        else "error"
+        if errors
+        else "healthy"
+        if indexed_at is not None
+        else "unknown"
+    )
+    return ExternalProviderRuntimeState(
+        provider_id=IMAP_PROVIDER_ID,
+        binding_ref=f"mail:profile:{profile.id}:imap",
+        authority_mode="external_mirror",
+        observed_at=observed_at,
+        configured=True,
+        active=active,
+        health=health,
+        freshness=freshness,
+        conflict="not_applicable",
+        recovery=(
+            "not_applicable"
+            if not active
+            else "ready"
+            if health == "healthy" and freshness == "current"
+            else "attention"
+        ),
+        last_success_at=indexed_at or _aware(metrics.get("last_bounce_success_at")),
+        detail=(
+            "IMAP mailbox access is disabled."
+            if not active
+            else "IMAP mailbox or bounce-source errors require attention."
+            if errors
+            else "IMAP mailbox state has not been indexed yet."
+            if indexed_at is None
+            else "IMAP mailbox index state is available."
+        ),
+        metrics={
+            "indexed_folders": int(metrics.get("indexed_folders", 0)),
+            "indexed_messages": int(metrics.get("indexed_messages", 0)),
+            "active_bounce_sources": int(metrics.get("active_bounce_sources", 0)),
+            "bounce_source_errors": errors,
+            "active_endpoints": sum(1 for item in endpoints if item.is_active),
+        },
+    )
+
+
+def _legacy_configured(profile: MailServerProfile, protocol: str) -> bool:
+    value = profile.smtp_config if protocol == "smtp" else profile.imap_config
+    return isinstance(value, dict) and bool(value)
+
+
+def _aware(value: object | None) -> datetime | None:
+    if not isinstance(value, datetime):
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value.astimezone(UTC)
+
+
+__all__ = [
+    "IMAP_PROVIDER_ID",
+    "SMTP_PROVIDER_ID",
+    "imap_provider_states",
+    "smtp_provider_states",
+]

tests/test_provider_state.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+import unittest
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from govoplan_access.backend.db import models as access_models  # noqa: F401
+from govoplan_core.core.provider_governance import ExternalProviderStateContext
+from govoplan_core.db.base import Base
+from govoplan_mail.backend.db.models import (
+    MailBounceSource,
+    MailDeliveryCommand,
+    MailMailboxFolderIndex,
+    MailMailboxMessageIndex,
+    MailServerEndpoint,
+    MailServerProfile,
+)
+from govoplan_mail.backend.manifest import manifest
+from govoplan_mail.backend.provider_state import (
+    IMAP_PROVIDER_ID,
+    SMTP_PROVIDER_ID,
+    imap_provider_states,
+    smtp_provider_states,
+)
+
+
+class MailProviderStateTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+        Base.metadata.create_all(
+            self.engine,
+            tables=(
+                MailServerProfile.__table__,
+                MailServerEndpoint.__table__,
+                MailDeliveryCommand.__table__,
+                MailMailboxFolderIndex.__table__,
+                MailMailboxMessageIndex.__table__,
+                MailBounceSource.__table__,
+            ),
+        )
+        self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
+        self.profile = MailServerProfile(
+            id="profile-1",
+            tenant_id="tenant-1",
+            scope_type="tenant",
+            scope_id="tenant-1",
+            name="Mail",
+            slug="mail",
+            is_active=True,
+            smtp_config={"host": "smtp.example.test", "port": 587},
+            imap_config={"host": "imap.example.test", "port": 993},
+        )
+        self.session.add(
+            self.profile
+        )
+        self.session.add(
+            MailMailboxFolderIndex(
+                id="folder-1",
+                tenant_id="tenant-1",
+                profile_id=self.profile.id,
+                folder="INBOX",
+                indexed_at=datetime.now(UTC),
+            )
+        )
+        self.session.commit()
+
+    def tearDown(self) -> None:
+        self.session.close()
+        self.engine.dispose()
+
+    def test_smtp_state_exposes_reconciliation_without_server_or_secrets(self) -> None:
+        self.session.add(
+            MailDeliveryCommand(
+                id="command-1",
+                tenant_id="tenant-1",
+                command_type="message",
+                source_module="notifications",
+                source_resource_type="notification",
+                idempotency_key="notification-1",
+                canonical_request_hash="a" * 64,
+                profile_id=self.profile.id,
+                expected_smtp_transport_revision="revision-1",
+                message_sha256="b" * 64,
+                message_size_bytes=10,
+                recipient_count=1,
+                status="outcome_unknown",
+                expires_at=datetime.now(UTC) + timedelta(days=1),
+            )
+        )
+        self.session.flush()
+
+        state = smtp_provider_states(
+            ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
+        )[0]
+
+        self.assertEqual("warning", state.health)
+        self.assertEqual("pending", state.conflict)
+        self.assertEqual("attention", state.recovery)
+        self.assertEqual(1, state.metrics["outcome_unknown_commands"])
+        self.assertNotIn("smtp.example.test", str(state.to_dict()))
+
+    def test_imap_state_is_fresh_tenant_bounded_and_registered(self) -> None:
+        state = imap_provider_states(
+            ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
+        )[0]
+
+        self.assertEqual("healthy", state.health)
+        self.assertEqual("current", state.freshness)
+        self.assertEqual("external_mirror", state.authority_mode)
+        self.assertNotIn("imap.example.test", str(state.to_dict()))
+        self.assertEqual(
+            (),
+            imap_provider_states(
+                ExternalProviderStateContext(
+                    session=self.session,
+                    tenant_id="tenant-2",
+                )
+            ),
+        )
+        self.assertEqual(
+            {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID},
+            {item.id for item in manifest.external_providers},
+        )
+        self.assertEqual(
+            {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID},
+            {
+                item.provider_id
+                for item in manifest.external_provider_state_providers
+            },
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()