from __future__ import annotations from collections import defaultdict from datetime import UTC, datetime, timedelta from typing import Any from sqlalchemy import case, func, or_, select from sqlalchemy.orm import Session from govoplan_core.core.provider_governance import ( ExternalProviderRuntimeState, ExternalProviderStateContext, ) from govoplan_mail.backend.db.models import ( MailBounceSource, MailDeliveryCommand, MailMailboxFolderIndex, MailMailboxMessageIndex, MailPop3Import, MailServerEndpoint, MailServerProfile, ) SMTP_PROVIDER_ID = "mail.smtp_delivery" IMAP_PROVIDER_ID = "mail.imap_mailbox" JMAP_PROVIDER_ID = "mail.jmap_mailbox" POP3_PROVIDER_ID = "mail.pop3_legacy_import" _CURRENT_INDEX_WINDOW = timedelta(minutes=30) def smtp_provider_states( context: ExternalProviderStateContext, ) -> tuple[ExternalProviderRuntimeState, ...]: return _mail_provider_states(context, protocol="smtp") def imap_provider_states( context: ExternalProviderStateContext, ) -> tuple[ExternalProviderRuntimeState, ...]: return _mail_provider_states(context, protocol="imap") def jmap_provider_states( context: ExternalProviderStateContext, ) -> tuple[ExternalProviderRuntimeState, ...]: return _mail_provider_states(context, protocol="jmap") def pop3_provider_states( context: ExternalProviderStateContext, ) -> tuple[ExternalProviderRuntimeState, ...]: if not isinstance(context.session, Session): raise RuntimeError("Mail provider state requires a database session.") profiles = _profiles(context) if not profiles: return () profile_ids = tuple(item.id for item in profiles) endpoints = _endpoints( context.session, profile_ids=profile_ids, protocol="pop3", ) endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list) for endpoint in endpoints: endpoints_by_profile[endpoint.profile_id].append(endpoint) metrics = _pop3_metrics( context.session, profile_ids=profile_ids, tenant_id=context.tenant_id, ) observed_at = datetime.now(UTC) return tuple( _pop3_state( profile, endpoints=endpoints_by_profile.get(profile.id, []), metrics=metrics.get(profile.id, {}), observed_at=observed_at, ) for profile in profiles if endpoints_by_profile.get(profile.id) ) def _mail_provider_states( context: ExternalProviderStateContext, *, protocol: str, ) -> tuple[ExternalProviderRuntimeState, ...]: if not isinstance(context.session, Session): raise RuntimeError("Mail provider state requires a database session.") profiles = _profiles(context) if not profiles: return () profile_ids = tuple(item.id for item in profiles) endpoints = _endpoints(context.session, profile_ids=profile_ids, protocol=protocol) endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list) for endpoint in endpoints: endpoints_by_profile[endpoint.profile_id].append(endpoint) observed_at = datetime.now(UTC) if protocol == "smtp": metrics = _smtp_metrics(context.session, profile_ids=profile_ids) return tuple( _smtp_state( profile, endpoints=endpoints_by_profile.get(profile.id, []), metrics=metrics.get(profile.id, {}), observed_at=observed_at, ) for profile in profiles if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "smtp") ) metrics = _imap_metrics(context.session, profile_ids=profile_ids) if protocol == "jmap": metrics = { profile_id: { key: value for key, value in values.items() if key in {"indexed_folders", "indexed_messages", "last_indexed_at"} } for profile_id, values in metrics.items() } provider_id = JMAP_PROVIDER_ID if protocol == "jmap" else IMAP_PROVIDER_ID return tuple( _imap_state( profile, endpoints=endpoints_by_profile.get(profile.id, []), metrics=metrics.get(profile.id, {}), observed_at=observed_at, protocol=protocol, provider_id=provider_id, ) for profile in profiles if endpoints_by_profile.get(profile.id) or (protocol == "imap" and _legacy_configured(profile, "imap")) ) def _profiles(context: ExternalProviderStateContext) -> tuple[MailServerProfile, ...]: statement = select(MailServerProfile) if context.tenant_id is not None: statement = statement.where( or_( MailServerProfile.tenant_id.is_(None), MailServerProfile.tenant_id == context.tenant_id, ) ) return tuple( context.session.scalars( statement.order_by( MailServerProfile.tenant_id, MailServerProfile.id, ).limit(context.max_items + 1) ) ) def _endpoints( session: Session, *, profile_ids: tuple[str, ...], protocol: str, ) -> tuple[MailServerEndpoint, ...]: return tuple( session.scalars( select(MailServerEndpoint).where( MailServerEndpoint.profile_id.in_(profile_ids), MailServerEndpoint.protocol == protocol, ) ) ) def _smtp_metrics( session: Session, *, profile_ids: tuple[str, ...], ) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = defaultdict(dict) rows = session.execute( select( MailDeliveryCommand.profile_id, MailDeliveryCommand.status, func.count(MailDeliveryCommand.id), func.max(MailDeliveryCommand.completed_at), ) .where(MailDeliveryCommand.profile_id.in_(profile_ids)) .group_by(MailDeliveryCommand.profile_id, MailDeliveryCommand.status) ) for profile_id, status, count, last_completed_at in rows: item = result[str(profile_id)] item[str(status)] = int(count) if status in {"accepted", "reconciled_accepted", "partially_refused"}: current = _aware(item.get("last_success_at")) candidate = _aware(last_completed_at) if candidate is not None and (current is None or candidate > current): item["last_success_at"] = candidate return result def _imap_metrics( session: Session, *, profile_ids: tuple[str, ...], ) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = defaultdict(dict) folder_rows = session.execute( select( MailMailboxFolderIndex.profile_id, func.count(MailMailboxFolderIndex.id), func.max(MailMailboxFolderIndex.indexed_at), ) .where(MailMailboxFolderIndex.profile_id.in_(profile_ids)) .group_by(MailMailboxFolderIndex.profile_id) ) for profile_id, count, indexed_at in folder_rows: result[str(profile_id)]["indexed_folders"] = int(count) result[str(profile_id)]["last_indexed_at"] = _aware(indexed_at) message_rows = session.execute( select( MailMailboxMessageIndex.profile_id, func.count(MailMailboxMessageIndex.id), ) .where(MailMailboxMessageIndex.profile_id.in_(profile_ids)) .group_by(MailMailboxMessageIndex.profile_id) ) for profile_id, count in message_rows: result[str(profile_id)]["indexed_messages"] = int(count) bounce_rows = session.execute( select( MailBounceSource.profile_id, func.count(MailBounceSource.id), func.sum( case((MailBounceSource.last_error.is_not(None), 1), else_=0) ), func.max(MailBounceSource.last_success_at), ) .where( MailBounceSource.profile_id.in_(profile_ids), MailBounceSource.is_active.is_(True), ) .group_by(MailBounceSource.profile_id) ) for profile_id, count, errors, last_success_at in bounce_rows: item = result[str(profile_id)] item["active_bounce_sources"] = int(count) item["bounce_source_errors"] = int(errors or 0) item["last_bounce_success_at"] = _aware(last_success_at) return result def _pop3_metrics( session: Session, *, profile_ids: tuple[str, ...], tenant_id: str | None, ) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = defaultdict(dict) statement = select( MailPop3Import.profile_id, MailPop3Import.deletion_status, func.count(MailPop3Import.id), func.max(MailPop3Import.imported_at), ).where(MailPop3Import.profile_id.in_(profile_ids)) if tenant_id is not None: statement = statement.where(MailPop3Import.tenant_id == tenant_id) rows = session.execute( statement.group_by( MailPop3Import.profile_id, MailPop3Import.deletion_status, ) ) for profile_id, deletion_status, count, last_imported_at in rows: item = result[str(profile_id)] item[f"deletion_{deletion_status}"] = int(count) current = _aware(item.get("last_imported_at")) candidate = _aware(last_imported_at) if candidate is not None and (current is None or candidate > current): item["last_imported_at"] = candidate return result def _smtp_state( profile: MailServerProfile, *, endpoints: list[MailServerEndpoint], metrics: dict[str, Any], observed_at: datetime, ) -> ExternalProviderRuntimeState: active = bool(profile.is_active) and ( any(item.is_active for item in endpoints) or (not endpoints and _legacy_configured(profile, "smtp")) ) outcome_unknown = int(metrics.get("outcome_unknown", 0)) last_success = _aware(metrics.get("last_success_at")) health = ( "inactive" if not active else "warning" if outcome_unknown else "healthy" if last_success is not None else "unknown" ) return ExternalProviderRuntimeState( provider_id=SMTP_PROVIDER_ID, binding_ref=f"mail:profile:{profile.id}:smtp", authority_mode="governance_overlay", observed_at=observed_at, configured=True, active=active, health=health, freshness="not_applicable", conflict="pending" if outcome_unknown else "clear", recovery=( "not_applicable" if not active else "attention" if outcome_unknown or last_success is None else "ready" ), last_success_at=last_success, detail=( "SMTP delivery is disabled." if not active else "SMTP outcomes require reconciliation." if outcome_unknown else "SMTP delivery has no retained successful observation yet." if last_success is None else "SMTP delivery evidence is available." ), metrics={ "pending_commands": int(metrics.get("pending", 0)), "temporary_failures": int(metrics.get("temporary_failure", 0)), "permanent_failures": int(metrics.get("permanent_failure", 0)), "outcome_unknown_commands": outcome_unknown, "active_endpoints": sum(1 for item in endpoints if item.is_active), }, ) def _imap_state( profile: MailServerProfile, *, endpoints: list[MailServerEndpoint], metrics: dict[str, Any], observed_at: datetime, protocol: str = "imap", provider_id: str = IMAP_PROVIDER_ID, ) -> ExternalProviderRuntimeState: protocol_label = protocol.upper() active = bool(profile.is_active) and ( any(item.is_active for item in endpoints) or (protocol == "imap" and not endpoints and _legacy_configured(profile, "imap")) ) indexed_at = _aware(metrics.get("last_indexed_at")) errors = int(metrics.get("bounce_source_errors", 0)) freshness = ( "not_applicable" if not active else "unknown" if indexed_at is None else "current" if observed_at - indexed_at <= _CURRENT_INDEX_WINDOW else "stale" ) health = ( "inactive" if not active else "error" if errors else "healthy" if indexed_at is not None else "unknown" ) return ExternalProviderRuntimeState( provider_id=provider_id, binding_ref=f"mail:profile:{profile.id}:{protocol}", authority_mode="external_mirror", observed_at=observed_at, configured=True, active=active, health=health, freshness=freshness, conflict="not_applicable", recovery=( "not_applicable" if not active else "ready" if health == "healthy" and freshness == "current" else "attention" ), last_success_at=indexed_at or _aware(metrics.get("last_bounce_success_at")), detail=( f"{protocol_label} mailbox access is disabled." if not active else f"{protocol_label} mailbox or bounce-source errors require attention." if errors else f"{protocol_label} mailbox state has not been indexed yet." if indexed_at is None else f"{protocol_label} mailbox index state is available." ), metrics={ "indexed_folders": int(metrics.get("indexed_folders", 0)), "indexed_messages": int(metrics.get("indexed_messages", 0)), "active_bounce_sources": int(metrics.get("active_bounce_sources", 0)), "bounce_source_errors": errors, "active_endpoints": sum(1 for item in endpoints if item.is_active), }, ) def _pop3_state( profile: MailServerProfile, *, endpoints: list[MailServerEndpoint], metrics: dict[str, Any], observed_at: datetime, ) -> ExternalProviderRuntimeState: enabled_endpoints = [ item for item in endpoints if item.is_active and bool((item.config or {}).get("legacy_import_enabled")) ] active = bool(profile.is_active) and bool(enabled_endpoints) failed_deletions = int(metrics.get("deletion_failed", 0)) unknown_deletions = int(metrics.get("deletion_outcome_unknown", 0)) last_imported_at = _aware(metrics.get("last_imported_at")) health = ( "inactive" if not active else "warning" if failed_deletions or unknown_deletions else "healthy" if last_imported_at is not None else "unknown" ) return ExternalProviderRuntimeState( provider_id=POP3_PROVIDER_ID, binding_ref=f"mail:profile:{profile.id}:pop3", authority_mode="governance_overlay", observed_at=observed_at, configured=True, active=active, health=health, freshness="not_applicable", conflict="pending" if unknown_deletions else "clear", recovery=( "not_applicable" if not active else "attention" if failed_deletions or unknown_deletions else "ready" ), last_success_at=last_imported_at, detail=( "POP3 legacy import is disabled." if not active else "POP3 source deletion evidence requires attention." if failed_deletions or unknown_deletions else "POP3 legacy import is enabled but has no retained import yet." if last_imported_at is None else "POP3 governed import evidence is available." ), metrics={ "active_endpoints": len(enabled_endpoints), "imports": sum( int(value) for key, value in metrics.items() if key.startswith("deletion_") ), "failed_deletions": failed_deletions, "outcome_unknown_deletions": unknown_deletions, }, ) def _legacy_configured(profile: MailServerProfile, protocol: str) -> bool: value = profile.smtp_config if protocol == "smtp" else profile.imap_config return isinstance(value, dict) and bool(value) def _aware(value: object | None) -> datetime | None: if not isinstance(value, datetime): return None if value.tzinfo is None: return value.replace(tzinfo=UTC) return value.astimezone(UTC) __all__ = [ "IMAP_PROVIDER_ID", "JMAP_PROVIDER_ID", "POP3_PROVIDER_ID", "SMTP_PROVIDER_ID", "imap_provider_states", "jmap_provider_states", "pop3_provider_states", "smtp_provider_states", ]