feat: declare governed external provider state

This commit is contained in:
2026-08-01 17:48:37 +02:00
parent fe309dbff5
commit 5a3b13e26e
4 changed files with 622 additions and 0 deletions
+6
View File
@@ -1,5 +1,11 @@
# GovOPlaN Mail Codex Guide
## Documentation Contract
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
- Keep feature content here; `govoplan-docs` projects it without importing Mail internals.
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
## Scope
This repository owns the `mail` module: SMTP/IMAP profiles, mail profile policy, encrypted mail credentials, SMTP sending, IMAP append and mailbox access, mock mail infrastructure, backend module manifest, and `@govoplan/mail-webui`.
+138
View File
@@ -28,12 +28,25 @@ from govoplan_core.core.modules import (
PermissionDefinition,
RoleTemplate,
)
from govoplan_core.core.provider_governance import (
ExternalProviderDeclaration,
ExternalProviderStateProviderRegistration,
ProviderBehaviorDeclaration,
ProviderObjectDeclaration,
declared_module_architecture,
)
from govoplan_core.core.views import ViewSurface
from govoplan_core.db.base import Base
from govoplan_mail.backend.documentation import (
documentation_configuration_states,
documentation_topics,
)
from govoplan_mail.backend.provider_state import (
IMAP_PROVIDER_ID,
SMTP_PROVIDER_ID,
imap_provider_states,
smtp_provider_states,
)
from govoplan_mail.backend.db import models as mail_models # noqa: F401 - populate Mail ORM metadata
@@ -188,6 +201,99 @@ def _mail_router(context: ModuleContext):
return aggregate
SMTP_PROVIDER = ExternalProviderDeclaration(
id=SMTP_PROVIDER_ID,
module_id="mail",
label="SMTP message delivery",
maturity="publish",
operations=("discover", "read", "publish", "preview", "dry_run"),
objects=(
ProviderObjectDeclaration(
object_type="mail_server_endpoint",
field_groups=("identity", "transport", "policy", "revision"),
authority_modes=("external_authoritative", "governance_overlay"),
default_authority_mode="governance_overlay",
),
ProviderObjectDeclaration(
object_type="outbound_message",
field_groups=("envelope", "content_digest", "delivery_state", "evidence"),
authority_modes=("governance_overlay",),
default_authority_mode="governance_overlay",
),
),
behavior=ProviderBehaviorDeclaration(
revision_tokens="Every command pins the selected SMTP endpoint, credential, and random transport revision.",
concurrency="The expected transport revision is checked before credentials are decrypted or an effect starts.",
freshness="Delivery outcomes and completion times are retained; transport freshness is not periodic state.",
health="Successful deliveries and unresolved outcome-unknown commands are projected without exposing server details.",
max_read_items=5000,
idempotency="Tenant, command type, and idempotency key bind one canonical encrypted delivery request.",
retry="Only classified pre-acceptance temporary failures are retried with bounded scheduling.",
timeout_seconds=60,
conflicts="A reused idempotency key with different content is rejected before delivery.",
outcome_unknown="A connection failure after effect start becomes outcome_unknown and is never blindly retried.",
outcome_unknown_supported=True,
evidence="Encrypted command, attempt, acceptance/refusal summary, effect-start marker, and reconciliation are retained.",
audit_event_types=(
"mail.delivery_requested",
"mail.delivery_completed",
"mail.delivery_reconciled",
),
correction="A reconciled new command is a separate auditable effect and does not rewrite the original outcome.",
rollback="SMTP acceptance cannot be rolled back.",
compensation="A follow-up message or domain correction is the only safe compensation after acceptance.",
reconciliation="Operators record provider evidence and choose accepted or not-accepted before any resend.",
outage="Pending commands remain durable; accepted or outcome-unknown commands are not redelivered automatically.",
classifications=("confidential", "personal", "special_category"),
purposes=("governed message delivery", "notification delivery"),
retention="Payload, delivery evidence, and audit records follow separate configured retention policies.",
secret_handling="Credentials are decrypted only inside Mail after authorization and revision validation.",
),
interface_names=("mail.campaign_delivery", "mail.delivery_commands", "mail.delivery_outbox"),
documentation_topic_ids=("mail.reference.campaign-delivery-contract",),
)
IMAP_PROVIDER = ExternalProviderDeclaration(
id=IMAP_PROVIDER_ID,
module_id="mail",
label="IMAP mailbox projection",
maturity="read",
operations=("discover", "search", "read", "preview"),
objects=(
ProviderObjectDeclaration(
object_type="mailbox_folder",
field_groups=("identity", "flags", "counts", "revision"),
authority_modes=("external_authoritative", "external_mirror"),
default_authority_mode="external_mirror",
),
ProviderObjectDeclaration(
object_type="mailbox_message",
field_groups=("identity", "headers", "body_preview", "flags", "source_metadata"),
authority_modes=("external_authoritative", "external_mirror"),
default_authority_mode="external_mirror",
),
),
behavior=ProviderBehaviorDeclaration(
revision_tokens="UIDVALIDITY, UID, folder, flags, and transport revision identify mailbox observations.",
concurrency="Bounded reads pin the profile transport revision and never mutate message flags.",
freshness="Folder and message index timestamps state when the external mailbox was last observed.",
health="Index state and bounce-source errors are projected independently of secret profile fields.",
max_read_items=500,
evidence="Mailbox index rows and bounce observations retain bounded source identities and observation times.",
audit_event_types=("mail.mailbox.read", "mail.bounce.observed"),
correction="A later mailbox refresh replaces the derived projection while source-owned history remains external.",
reconciliation="UIDVALIDITY changes invalidate the affected derived index before a bounded refresh.",
outage="The last derived index remains readable with stale or unknown freshness where policy permits.",
classifications=("confidential", "personal", "special_category"),
purposes=("mailbox access", "delivery-status processing"),
retention="Derived mailbox indexes and bounce evidence follow Mail retention policy.",
secret_handling="IMAP credentials remain encrypted and are never returned through mailbox or provider-state APIs.",
),
documentation_topic_ids=("mail.workflow.read-mailbox",),
)
manifest = ModuleManifest(
id="mail",
name="Mail",
@@ -563,6 +669,38 @@ manifest = ModuleManifest(
resolve=documentation_configuration_states,
),
),
external_providers=(SMTP_PROVIDER, IMAP_PROVIDER),
external_provider_state_providers=(
ExternalProviderStateProviderRegistration(
module_id="mail",
provider_id=SMTP_PROVIDER_ID,
provider=smtp_provider_states,
),
ExternalProviderStateProviderRegistration(
module_id="mail",
provider_id=IMAP_PROVIDER_ID,
provider=imap_provider_states,
),
),
architecture=declared_module_architecture(
layer="communication_participation",
kind="integration",
maturity="vertical_slice",
documentation_ref="docs/MAIL_HANDBOOK.md",
test_ref="tests/test_delivery_outbox.py",
known_limits=("Provider recovery drills and a complete webmail profile are not reference-ready.",),
supported_authority_modes=(
"external_authoritative",
"external_mirror",
"governance_overlay",
),
owned_concepts=("mail profile", "mail delivery command", "delivery attempt", "mailbox projection"),
non_owned_concepts=("campaign", "notification", "recipient address directory", "external mailbox"),
target_tested_providers=(SMTP_PROVIDER_ID, IMAP_PROVIDER_ID),
recovery_docs=("docs/MAIL_HANDBOOK.md",),
security_docs=("docs/MAIL_HANDBOOK.md",),
operations_docs=("docs/MAIL_HANDBOOK.md",),
),
)
+342
View File
@@ -0,0 +1,342 @@
from __future__ import annotations
from collections import defaultdict
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import case, func, or_, select
from sqlalchemy.orm import Session
from govoplan_core.core.provider_governance import (
ExternalProviderRuntimeState,
ExternalProviderStateContext,
)
from govoplan_mail.backend.db.models import (
MailBounceSource,
MailDeliveryCommand,
MailMailboxFolderIndex,
MailMailboxMessageIndex,
MailServerEndpoint,
MailServerProfile,
)
SMTP_PROVIDER_ID = "mail.smtp_delivery"
IMAP_PROVIDER_ID = "mail.imap_mailbox"
_CURRENT_INDEX_WINDOW = timedelta(minutes=30)
def smtp_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
return _mail_provider_states(context, protocol="smtp")
def imap_provider_states(
context: ExternalProviderStateContext,
) -> tuple[ExternalProviderRuntimeState, ...]:
return _mail_provider_states(context, protocol="imap")
def _mail_provider_states(
context: ExternalProviderStateContext,
*,
protocol: str,
) -> tuple[ExternalProviderRuntimeState, ...]:
if not isinstance(context.session, Session):
raise RuntimeError("Mail provider state requires a database session.")
profiles = _profiles(context)
if not profiles:
return ()
profile_ids = tuple(item.id for item in profiles)
endpoints = _endpoints(context.session, profile_ids=profile_ids, protocol=protocol)
endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list)
for endpoint in endpoints:
endpoints_by_profile[endpoint.profile_id].append(endpoint)
observed_at = datetime.now(UTC)
if protocol == "smtp":
metrics = _smtp_metrics(context.session, profile_ids=profile_ids)
return tuple(
_smtp_state(
profile,
endpoints=endpoints_by_profile.get(profile.id, []),
metrics=metrics.get(profile.id, {}),
observed_at=observed_at,
)
for profile in profiles
if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "smtp")
)
metrics = _imap_metrics(context.session, profile_ids=profile_ids)
return tuple(
_imap_state(
profile,
endpoints=endpoints_by_profile.get(profile.id, []),
metrics=metrics.get(profile.id, {}),
observed_at=observed_at,
)
for profile in profiles
if endpoints_by_profile.get(profile.id) or _legacy_configured(profile, "imap")
)
def _profiles(context: ExternalProviderStateContext) -> tuple[MailServerProfile, ...]:
statement = select(MailServerProfile)
if context.tenant_id is not None:
statement = statement.where(
or_(
MailServerProfile.tenant_id.is_(None),
MailServerProfile.tenant_id == context.tenant_id,
)
)
return tuple(
context.session.scalars(
statement.order_by(
MailServerProfile.tenant_id,
MailServerProfile.id,
).limit(context.max_items + 1)
)
)
def _endpoints(
session: Session,
*,
profile_ids: tuple[str, ...],
protocol: str,
) -> tuple[MailServerEndpoint, ...]:
return tuple(
session.scalars(
select(MailServerEndpoint).where(
MailServerEndpoint.profile_id.in_(profile_ids),
MailServerEndpoint.protocol == protocol,
)
)
)
def _smtp_metrics(
session: Session,
*,
profile_ids: tuple[str, ...],
) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = defaultdict(dict)
rows = session.execute(
select(
MailDeliveryCommand.profile_id,
MailDeliveryCommand.status,
func.count(MailDeliveryCommand.id),
func.max(MailDeliveryCommand.completed_at),
)
.where(MailDeliveryCommand.profile_id.in_(profile_ids))
.group_by(MailDeliveryCommand.profile_id, MailDeliveryCommand.status)
)
for profile_id, status, count, last_completed_at in rows:
item = result[str(profile_id)]
item[str(status)] = int(count)
if status in {"accepted", "reconciled_accepted", "partially_refused"}:
current = _aware(item.get("last_success_at"))
candidate = _aware(last_completed_at)
if candidate is not None and (current is None or candidate > current):
item["last_success_at"] = candidate
return result
def _imap_metrics(
session: Session,
*,
profile_ids: tuple[str, ...],
) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = defaultdict(dict)
folder_rows = session.execute(
select(
MailMailboxFolderIndex.profile_id,
func.count(MailMailboxFolderIndex.id),
func.max(MailMailboxFolderIndex.indexed_at),
)
.where(MailMailboxFolderIndex.profile_id.in_(profile_ids))
.group_by(MailMailboxFolderIndex.profile_id)
)
for profile_id, count, indexed_at in folder_rows:
result[str(profile_id)]["indexed_folders"] = int(count)
result[str(profile_id)]["last_indexed_at"] = _aware(indexed_at)
message_rows = session.execute(
select(
MailMailboxMessageIndex.profile_id,
func.count(MailMailboxMessageIndex.id),
)
.where(MailMailboxMessageIndex.profile_id.in_(profile_ids))
.group_by(MailMailboxMessageIndex.profile_id)
)
for profile_id, count in message_rows:
result[str(profile_id)]["indexed_messages"] = int(count)
bounce_rows = session.execute(
select(
MailBounceSource.profile_id,
func.count(MailBounceSource.id),
func.sum(
case((MailBounceSource.last_error.is_not(None), 1), else_=0)
),
func.max(MailBounceSource.last_success_at),
)
.where(
MailBounceSource.profile_id.in_(profile_ids),
MailBounceSource.is_active.is_(True),
)
.group_by(MailBounceSource.profile_id)
)
for profile_id, count, errors, last_success_at in bounce_rows:
item = result[str(profile_id)]
item["active_bounce_sources"] = int(count)
item["bounce_source_errors"] = int(errors or 0)
item["last_bounce_success_at"] = _aware(last_success_at)
return result
def _smtp_state(
profile: MailServerProfile,
*,
endpoints: list[MailServerEndpoint],
metrics: dict[str, Any],
observed_at: datetime,
) -> ExternalProviderRuntimeState:
active = bool(profile.is_active) and (
any(item.is_active for item in endpoints)
or (not endpoints and _legacy_configured(profile, "smtp"))
)
outcome_unknown = int(metrics.get("outcome_unknown", 0))
last_success = _aware(metrics.get("last_success_at"))
health = (
"inactive"
if not active
else "warning"
if outcome_unknown
else "healthy"
if last_success is not None
else "unknown"
)
return ExternalProviderRuntimeState(
provider_id=SMTP_PROVIDER_ID,
binding_ref=f"mail:profile:{profile.id}:smtp",
authority_mode="governance_overlay",
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness="not_applicable",
conflict="pending" if outcome_unknown else "clear",
recovery=(
"not_applicable"
if not active
else "attention"
if outcome_unknown or last_success is None
else "ready"
),
last_success_at=last_success,
detail=(
"SMTP delivery is disabled."
if not active
else "SMTP outcomes require reconciliation."
if outcome_unknown
else "SMTP delivery has no retained successful observation yet."
if last_success is None
else "SMTP delivery evidence is available."
),
metrics={
"pending_commands": int(metrics.get("pending", 0)),
"temporary_failures": int(metrics.get("temporary_failure", 0)),
"permanent_failures": int(metrics.get("permanent_failure", 0)),
"outcome_unknown_commands": outcome_unknown,
"active_endpoints": sum(1 for item in endpoints if item.is_active),
},
)
def _imap_state(
profile: MailServerProfile,
*,
endpoints: list[MailServerEndpoint],
metrics: dict[str, Any],
observed_at: datetime,
) -> ExternalProviderRuntimeState:
active = bool(profile.is_active) and (
any(item.is_active for item in endpoints)
or (not endpoints and _legacy_configured(profile, "imap"))
)
indexed_at = _aware(metrics.get("last_indexed_at"))
errors = int(metrics.get("bounce_source_errors", 0))
freshness = (
"not_applicable"
if not active
else "unknown"
if indexed_at is None
else "current"
if observed_at - indexed_at <= _CURRENT_INDEX_WINDOW
else "stale"
)
health = (
"inactive"
if not active
else "error"
if errors
else "healthy"
if indexed_at is not None
else "unknown"
)
return ExternalProviderRuntimeState(
provider_id=IMAP_PROVIDER_ID,
binding_ref=f"mail:profile:{profile.id}:imap",
authority_mode="external_mirror",
observed_at=observed_at,
configured=True,
active=active,
health=health,
freshness=freshness,
conflict="not_applicable",
recovery=(
"not_applicable"
if not active
else "ready"
if health == "healthy" and freshness == "current"
else "attention"
),
last_success_at=indexed_at or _aware(metrics.get("last_bounce_success_at")),
detail=(
"IMAP mailbox access is disabled."
if not active
else "IMAP mailbox or bounce-source errors require attention."
if errors
else "IMAP mailbox state has not been indexed yet."
if indexed_at is None
else "IMAP mailbox index state is available."
),
metrics={
"indexed_folders": int(metrics.get("indexed_folders", 0)),
"indexed_messages": int(metrics.get("indexed_messages", 0)),
"active_bounce_sources": int(metrics.get("active_bounce_sources", 0)),
"bounce_source_errors": errors,
"active_endpoints": sum(1 for item in endpoints if item.is_active),
},
)
def _legacy_configured(profile: MailServerProfile, protocol: str) -> bool:
value = profile.smtp_config if protocol == "smtp" else profile.imap_config
return isinstance(value, dict) and bool(value)
def _aware(value: object | None) -> datetime | None:
if not isinstance(value, datetime):
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
__all__ = [
"IMAP_PROVIDER_ID",
"SMTP_PROVIDER_ID",
"imap_provider_states",
"smtp_provider_states",
]
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db import models as access_models # noqa: F401
from govoplan_core.core.provider_governance import ExternalProviderStateContext
from govoplan_core.db.base import Base
from govoplan_mail.backend.db.models import (
MailBounceSource,
MailDeliveryCommand,
MailMailboxFolderIndex,
MailMailboxMessageIndex,
MailServerEndpoint,
MailServerProfile,
)
from govoplan_mail.backend.manifest import manifest
from govoplan_mail.backend.provider_state import (
IMAP_PROVIDER_ID,
SMTP_PROVIDER_ID,
imap_provider_states,
smtp_provider_states,
)
class MailProviderStateTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
Base.metadata.create_all(
self.engine,
tables=(
MailServerProfile.__table__,
MailServerEndpoint.__table__,
MailDeliveryCommand.__table__,
MailMailboxFolderIndex.__table__,
MailMailboxMessageIndex.__table__,
MailBounceSource.__table__,
),
)
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
self.profile = MailServerProfile(
id="profile-1",
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
name="Mail",
slug="mail",
is_active=True,
smtp_config={"host": "smtp.example.test", "port": 587},
imap_config={"host": "imap.example.test", "port": 993},
)
self.session.add(
self.profile
)
self.session.add(
MailMailboxFolderIndex(
id="folder-1",
tenant_id="tenant-1",
profile_id=self.profile.id,
folder="INBOX",
indexed_at=datetime.now(UTC),
)
)
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_smtp_state_exposes_reconciliation_without_server_or_secrets(self) -> None:
self.session.add(
MailDeliveryCommand(
id="command-1",
tenant_id="tenant-1",
command_type="message",
source_module="notifications",
source_resource_type="notification",
idempotency_key="notification-1",
canonical_request_hash="a" * 64,
profile_id=self.profile.id,
expected_smtp_transport_revision="revision-1",
message_sha256="b" * 64,
message_size_bytes=10,
recipient_count=1,
status="outcome_unknown",
expires_at=datetime.now(UTC) + timedelta(days=1),
)
)
self.session.flush()
state = smtp_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual("warning", state.health)
self.assertEqual("pending", state.conflict)
self.assertEqual("attention", state.recovery)
self.assertEqual(1, state.metrics["outcome_unknown_commands"])
self.assertNotIn("smtp.example.test", str(state.to_dict()))
def test_imap_state_is_fresh_tenant_bounded_and_registered(self) -> None:
state = imap_provider_states(
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
)[0]
self.assertEqual("healthy", state.health)
self.assertEqual("current", state.freshness)
self.assertEqual("external_mirror", state.authority_mode)
self.assertNotIn("imap.example.test", str(state.to_dict()))
self.assertEqual(
(),
imap_provider_states(
ExternalProviderStateContext(
session=self.session,
tenant_id="tenant-2",
)
),
)
self.assertEqual(
{SMTP_PROVIDER_ID, IMAP_PROVIDER_ID},
{item.id for item in manifest.external_providers},
)
self.assertEqual(
{SMTP_PROVIDER_ID, IMAP_PROVIDER_ID},
{
item.provider_id
for item in manifest.external_provider_state_providers
},
)
if __name__ == "__main__":
unittest.main()