feat(mail): add governed POP3 legacy import

src/govoplan_mail/backend/router.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 import dataclasses
+import hashlib
 from types import SimpleNamespace
 from typing import Any
 
 from fastapi import APIRouter, Depends, HTTPException, Query, status
-from sqlalchemy import func, or_
+from sqlalchemy import func, or_, select
 from sqlalchemy.orm import Session
 
 from govoplan_mail.backend.schemas import (
@@ -43,6 +44,13 @@ from govoplan_mail.backend.schemas import (
     MailProfilePolicyResponse,
     MailProfilePolicyUpdateRequest,
     MailSettingsDeltaResponse,
+    MailPop3ImportListResponse,
+    MailPop3ImportRecordResponse,
+    MailPop3ImportRequest,
+    MailPop3ImportResponse,
+    MailPop3MessagePreviewResponse,
+    MailPop3PreviewRequest,
+    MailPop3PreviewResponse,
     MailServerProfileCreateRequest,
     MailServerEndpointCreateRequest,
     MailServerEndpointResponse,
@@ -91,7 +99,15 @@ from govoplan_mail.backend.mail_profiles import (
     smtp_config_from_profile,
     update_mail_server_profile,
 )
-from govoplan_mail.backend.config import ImapConfig, SmtpConfig
+from govoplan_mail.backend.config import ImapConfig, Pop3Config, SmtpConfig
+from govoplan_mail.backend.db.models import MailPop3Import
+from govoplan_mail.backend.pop3_imports import (
+    Pop3ImportError,
+    create_pop3_imports,
+    list_pop3_imports,
+    mark_pop3_deletion_result,
+    pop3_import_payload,
+)
 from govoplan_mail.backend.runtime import get_registry
 from govoplan_mail.backend.recovery import (
     MailRecoveryError,
@@ -139,6 +155,14 @@ from govoplan_mail.backend.server_hierarchy import (
 )
 from govoplan_mail.backend.sending.imap import ImapAppendError, ImapConfigurationError, get_imap_message, list_imap_folders, list_imap_messages, load_imap_mailbox_bootstrap, test_imap_login
 from govoplan_mail.backend.sending.smtp import test_smtp_login
+from govoplan_mail.backend.sending.pop3 import (
+    Pop3ConfigurationError,
+    Pop3ProviderError,
+    delete_pop3_messages,
+    download_pop3_messages,
+    preview_pop3_messages,
+    test_pop3_login,
+)
 
 router = APIRouter(prefix="/mail", tags=["mail"])
 
@@ -1593,6 +1617,8 @@ def create_profile_server(
     principal: ApiPrincipal = Depends(get_api_principal),
     session: Session = Depends(get_session),
 ):
+    if payload.protocol == "pop3":
+        _require_scope(principal, "mail:pop3:manage")
     try:
         profile = _profile_for_mutation(
             session,
@@ -1652,6 +1678,8 @@ def update_profile_server(
             profile_id=profile_id,
             server_id=server_id,
         )
+        if server.protocol == "pop3":
+            _require_scope(principal, "mail:pop3:manage")
         update_mail_server_endpoint(
             session,
             server=server,
@@ -1703,6 +1731,8 @@ def deactivate_profile_server(
             profile_id=profile_id,
             server_id=server_id,
         )
+        if server.protocol == "pop3":
+            _require_scope(principal, "mail:pop3:manage")
         update_mail_server_endpoint(
             session,
             server=server,
@@ -2666,6 +2696,359 @@ def test_profile_imap(
         return MailConnectionTestResponse(ok=False, protocol="imap", message=_safe_error_message(exc), details={"error_type": exc.__class__.__name__})
 
 
+def _resolve_profile_pop3_transport(
+    session: Session,
+    *,
+    principal: ApiPrincipal,
+    profile_id: str,
+    server_id: str,
+    credential_id: str | None,
+):
+    profile = _get_profile_for_principal(
+        session,
+        principal=principal,
+        profile_id=profile_id,
+        require_active=True,
+    )
+    resolved = resolve_mail_transport(
+        session,
+        profile=profile,
+        protocol="pop3",
+        context=_transport_context_for_principal(
+            session,
+            principal=principal,
+        ),
+        server_id=server_id,
+        credential_id=credential_id,
+    )
+    if resolved.server is None or not isinstance(resolved.config, Pop3Config):
+        raise MailServerHierarchyError("The selected POP3 server is unavailable")
+    return profile, resolved
+
+
+@router.post(
+    "/profiles/{profile_id}/test-pop3",
+    response_model=MailConnectionTestResponse,
+)
+def test_profile_pop3(
+    profile_id: str,
+    server_id: str = Query(...),
+    credential_id: str | None = Query(default=None),
+    principal: ApiPrincipal = Depends(get_api_principal),
+    session: Session = Depends(get_session),
+):
+    _require_scope(principal, "mail:profile:test")
+    _require_scope(principal, "mail:profile:use")
+    _require_any_scope(principal, "mail:pop3:manage", "mail:pop3:import")
+    try:
+        _profile, resolved = _resolve_profile_pop3_transport(
+            session,
+            principal=principal,
+            profile_id=profile_id,
+            server_id=server_id,
+            credential_id=credential_id,
+        )
+        result = test_pop3_login(pop3_config=resolved.config)
+        return MailConnectionTestResponse(
+            ok=True,
+            protocol="pop3",
+            host=result.host,
+            port=result.port,
+            security=result.security,
+            message="POP3 connection successful.",
+            details={
+                "authenticated": result.authenticated,
+                "message_count": result.message_count,
+                "mailbox_size_bytes": result.mailbox_size_bytes,
+                "legacy_import": True,
+            },
+        )
+    except (MailProfileError, MailServerHierarchyError) as exc:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=str(exc),
+        ) from exc
+    except (Pop3ConfigurationError, Pop3ProviderError) as exc:
+        return MailConnectionTestResponse(
+            ok=False,
+            protocol="pop3",
+            message=_safe_error_message(exc),
+            details={"error_type": exc.__class__.__name__, "legacy_import": True},
+        )
+
+
+@router.post(
+    "/profiles/{profile_id}/pop3/preview",
+    response_model=MailPop3PreviewResponse,
+)
+def preview_profile_pop3_import(
+    profile_id: str,
+    payload: MailPop3PreviewRequest,
+    principal: ApiPrincipal = Depends(get_api_principal),
+    session: Session = Depends(get_session),
+):
+    _require_scope(principal, "mail:profile:use")
+    _require_scope(principal, "mail:pop3:import")
+    try:
+        _profile, resolved = _resolve_profile_pop3_transport(
+            session,
+            principal=principal,
+            profile_id=profile_id,
+            server_id=payload.server_id,
+            credential_id=payload.credential_id,
+        )
+        result = preview_pop3_messages(
+            pop3_config=resolved.config,
+            limit=payload.limit,
+        )
+        uidls = [message.uidl for message in result.messages]
+        imported_uidls = set(
+            session.scalars(
+                select(MailPop3Import.provider_uidl).where(
+                    MailPop3Import.tenant_id == principal.tenant_id,
+                    MailPop3Import.profile_id == profile_id,
+                    MailPop3Import.pop3_server_id == resolved.server.id,
+                    MailPop3Import.provider_uidl.in_(uidls),
+                )
+            )
+        ) if uidls else set()
+        return MailPop3PreviewResponse(
+            profile_id=profile_id,
+            server_id=resolved.server.id,
+            transport_revision=resolved.transport_revision,
+            host=result.host,
+            port=result.port,
+            security=result.security,
+            message_count=result.message_count,
+            mailbox_size_bytes=result.mailbox_size_bytes,
+            delete_after_import_allowed=resolved.config.allow_delete_after_import,
+            messages=[
+                MailPop3MessagePreviewResponse(
+                    message_number=message.message_number,
+                    uidl=message.uidl,
+                    subject=message.subject,
+                    from_header=message.from_header,
+                    to_header=message.to_header,
+                    date=message.date,
+                    message_id=message.message_id,
+                    size_bytes=message.size_bytes,
+                    body_preview=message.body_preview,
+                    already_imported=message.uidl in imported_uidls,
+                )
+                for message in result.messages
+            ],
+        )
+    except (MailProfileError, MailServerHierarchyError) as exc:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=str(exc),
+        ) from exc
+    except Pop3ConfigurationError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=str(exc),
+        ) from exc
+    except Pop3ProviderError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=str(exc),
+        ) from exc
+
+
+@router.post(
+    "/profiles/{profile_id}/pop3/import",
+    response_model=MailPop3ImportResponse,
+)
+def import_profile_pop3_messages(
+    profile_id: str,
+    payload: MailPop3ImportRequest,
+    principal: ApiPrincipal = Depends(get_api_principal),
+    session: Session = Depends(get_session),
+):
+    _require_scope(principal, "mail:profile:use")
+    _require_scope(principal, "mail:pop3:import")
+    if payload.delete_after_import:
+        _require_scope(principal, "mail:pop3:delete")
+    try:
+        _profile, resolved = _resolve_profile_pop3_transport(
+            session,
+            principal=principal,
+            profile_id=profile_id,
+            server_id=payload.server_id,
+            credential_id=payload.credential_id,
+        )
+        if resolved.transport_revision != payload.expected_transport_revision:
+            raise HTTPException(
+                status_code=status.HTTP_409_CONFLICT,
+                detail="The POP3 server or credential selection changed; refresh the preview before importing",
+            )
+        if payload.delete_after_import and not resolved.config.allow_delete_after_import:
+            raise HTTPException(
+                status_code=status.HTTP_409_CONFLICT,
+                detail="Delete-after-import is disabled for the selected POP3 server",
+            )
+        downloaded = download_pop3_messages(
+            pop3_config=resolved.config,
+            uidls=payload.uidls,
+        )
+        imported = create_pop3_imports(
+            session,
+            tenant_id=principal.tenant_id,
+            profile_id=profile_id,
+            pop3_server_id=resolved.server.id,
+            pop3_credential_id=(
+                resolved.credential.id if resolved.credential is not None else None
+            ),
+            transport_revision=resolved.transport_revision,
+            messages=downloaded,
+            user_id=principal.user.id,
+            deletion_requested=payload.delete_after_import,
+        )
+        aggregate_digest = hashlib.sha256(
+            "|".join(sorted(item.raw_sha256 for item in downloaded)).encode("ascii")
+        ).hexdigest()
+        audit_event(
+            session,
+            tenant_id=principal.tenant_id,
+            user_id=principal.user.id,
+            action="mail.pop3.imported",
+            object_type="mail_pop3_import_batch",
+            object_id=aggregate_digest,
+            details={
+                "profile_id": profile_id,
+                "server_id": resolved.server.id,
+                "transport_revision": resolved.transport_revision,
+                "selected_count": len(downloaded),
+                "imported_count": len(imported.imported),
+                "duplicate_count": len(imported.duplicates),
+                "delete_after_import": payload.delete_after_import,
+                "content_digest": aggregate_digest,
+            },
+        )
+        # The governed local copy and its audit evidence become durable before
+        # any separately authorized provider deletion is attempted.
+        session.commit()
+
+        deletion_status = "not_requested"
+        if payload.delete_after_import:
+            if not imported.imported:
+                deletion_status = "skipped_no_new_messages"
+            else:
+                new_uidls = [row.provider_uidl for row in imported.imported]
+                try:
+                    delete_pop3_messages(
+                        pop3_config=resolved.config,
+                        uidls=new_uidls,
+                    )
+                    deletion_status = "succeeded"
+                    deletion_error = None
+                except Pop3ProviderError as exc:
+                    deletion_status = (
+                        "outcome_unknown" if exc.outcome_unknown else "failed"
+                    )
+                    deletion_error = str(exc)
+                rows = mark_pop3_deletion_result(
+                    session,
+                    tenant_id=principal.tenant_id,
+                    import_ids=[row.id for row in imported.imported],
+                    status=deletion_status,
+                    error=deletion_error,
+                )
+                # Persist the provider outcome independently of the audit
+                # projection. If audit insertion is unavailable after the
+                # irreversible provider operation, the import record still
+                # retains the result for reconciliation and the pre-effect
+                # import audit already proves that deletion was requested.
+                session.commit()
+                audit_event(
+                    session,
+                    tenant_id=principal.tenant_id,
+                    user_id=principal.user.id,
+                    action="mail.pop3.source_deletion",
+                    object_type="mail_pop3_import_batch",
+                    object_id=aggregate_digest,
+                    details={
+                        "profile_id": profile_id,
+                        "server_id": resolved.server.id,
+                        "import_count": len(rows),
+                        "status": deletion_status,
+                    },
+                )
+                session.commit()
+
+        return MailPop3ImportResponse(
+            imports=[
+                MailPop3ImportRecordResponse.model_validate(
+                    pop3_import_payload(row)
+                )
+                for row in imported.imported
+            ],
+            duplicate_uidls=sorted(row.provider_uidl for row in imported.duplicates),
+            deletion_status=deletion_status,
+        )
+    except HTTPException:
+        session.rollback()
+        raise
+    except (MailProfileError, MailServerHierarchyError) as exc:
+        session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=str(exc),
+        ) from exc
+    except (Pop3ConfigurationError, Pop3ImportError) as exc:
+        session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=str(exc),
+        ) from exc
+    except Pop3ProviderError as exc:
+        session.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=str(exc),
+        ) from exc
+    except Exception:
+        session.rollback()
+        raise
+
+
+@router.get("/pop3/imports", response_model=MailPop3ImportListResponse)
+def get_pop3_imports(
+    profile_id: str | None = Query(default=None),
+    limit: int = Query(default=100, ge=1, le=500),
+    principal: ApiPrincipal = Depends(get_api_principal),
+    session: Session = Depends(get_session),
+):
+    _require_scope(principal, "mail:pop3:import")
+    visible_profile_ids = {
+        profile.id
+        for profile in list_mail_server_profiles(
+            session,
+            tenant_id=principal.tenant_id,
+            include_inactive=True,
+            **_profile_actor_kwargs(principal, administrative_visibility=True),
+        )
+    }
+    if profile_id and profile_id not in visible_profile_ids:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Mail-server profile not found",
+        )
+    rows = list_pop3_imports(
+        session,
+        tenant_id=principal.tenant_id,
+        profile_id=profile_id,
+        profile_ids=visible_profile_ids,
+        limit=limit,
+    )
+    return MailPop3ImportListResponse(
+        imports=[
+            MailPop3ImportRecordResponse.model_validate(pop3_import_payload(row))
+            for row in rows
+        ]
+    )
+
+
 @router.post("/profiles/{profile_id}/list-imap-folders", response_model=MailImapFolderListResponse)
 def list_profile_imap_folders(
     profile_id: str,