feat(mail): add governed JMAP mailbox sync and search
Module Package Release / publish-packages (push) Successful in 12s

This commit is contained in:
2026-08-22 17:09:05 +02:00
parent fd808336bc
commit 1cbf4acaf4
29 changed files with 2481 additions and 183 deletions
+482 -3
View File
@@ -35,6 +35,7 @@ from govoplan_mail.backend.schemas import (
MailDeliveryReconcileRequest,
MailDeliveryResendRequest,
MailMailboxBootstrapResponse,
MailMailboxChangesResponse,
MailImapTestRequest,
MailMailboxAttachmentResponse,
MailMailboxMessageDetailResponse,
@@ -81,6 +82,7 @@ from govoplan_mail.backend.mailbox_index import (
)
from govoplan_mail.backend.mail_profiles import (
MailProfileError,
assert_mail_policy_allows_transport,
campaign_mail_owner_context,
campaign_mail_context_visible_to_actor,
create_mail_server_profile,
@@ -99,7 +101,7 @@ from govoplan_mail.backend.mail_profiles import (
smtp_config_from_profile,
update_mail_server_profile,
)
from govoplan_mail.backend.config import ImapConfig, Pop3Config, SmtpConfig
from govoplan_mail.backend.config import ImapConfig, JmapConfig, Pop3Config, SmtpConfig
from govoplan_mail.backend.db.models import MailPop3Import
from govoplan_mail.backend.pop3_imports import (
Pop3ImportError,
@@ -154,6 +156,19 @@ from govoplan_mail.backend.server_hierarchy import (
update_mail_server_endpoint,
)
from govoplan_mail.backend.sending.imap import ImapAppendError, ImapConfigurationError, get_imap_message, list_imap_folders, list_imap_messages, load_imap_mailbox_bootstrap, test_imap_login
from govoplan_mail.backend.sending.jmap import (
JmapAuthenticationError,
JmapCapabilityError,
JmapConfigurationError,
JmapPermissionError,
JmapProviderError,
get_jmap_email_changes,
get_jmap_message,
list_jmap_folders,
list_jmap_messages,
load_jmap_mailbox_bootstrap,
test_jmap_connection,
)
from govoplan_mail.backend.sending.smtp import test_smtp_login
from govoplan_mail.backend.sending.pop3 import (
Pop3ConfigurationError,
@@ -175,6 +190,7 @@ MAIL_POLICY_RESOURCE = "mail_profile_policy"
MAIL_SERVER_RESOURCE = "mail_server"
MAIL_CREDENTIAL_RESOURCE = "mail_credential"
MAILBOX_MESSAGES_CURSOR_SCOPE = "mail.mailbox.messages.v1"
JMAP_MAILBOX_MESSAGES_CURSOR_SCOPE = "mail.mailbox.jmap.messages.v1"
DEFAULT_MAILBOX_MESSAGE_LIMIT = 50
CAPABILITY_ADDRESSES_LOOKUP = "addresses.lookup"
CAPABILITY_ADDRESSES_CONTACT_WRITER = "addresses.contact_writer"
@@ -1201,6 +1217,105 @@ def _next_mailbox_cursor(
)
def _jmap_mailbox_cursor_fingerprint(
*,
tenant_id: str,
profile_id: str,
folder: str,
limit: int,
query: str,
) -> str:
return keyset_query_fingerprint(
JMAP_MAILBOX_MESSAGES_CURSOR_SCOPE,
{
"tenant_id": tenant_id,
"profile_id": profile_id,
"folder": folder,
"limit": limit,
"query": query,
"protocol": "jmap",
}
)
def _jmap_mailbox_cursor_position(
cursor: str | None,
*,
fingerprint: str,
offset: int,
) -> tuple[int, str | None]:
if not cursor:
return offset, None
try:
values = decode_keyset_cursor(
JMAP_MAILBOX_MESSAGES_CURSOR_SCOPE,
cursor,
fingerprint=fingerprint,
)
except KeysetCursorError as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=str(exc),
) from exc
raw_offset = values.get("offset")
query_state = values.get("query_state")
if not isinstance(raw_offset, int) or raw_offset < 0:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid JMAP pagination cursor")
if not isinstance(query_state, str) or not query_state:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid JMAP pagination cursor")
return raw_offset, query_state
def _jmap_mailbox_cursor_limit(cursor: str | None, explicit_limit: int | None) -> int:
if explicit_limit is not None:
return explicit_limit
if not cursor:
return DEFAULT_MAILBOX_MESSAGE_LIMIT
try:
values = decode_keyset_cursor(JMAP_MAILBOX_MESSAGES_CURSOR_SCOPE, cursor)
except KeysetCursorError as exc:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
raw_limit = values.get("limit")
if not isinstance(raw_limit, int) or raw_limit < 1 or raw_limit > 100:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid JMAP pagination cursor")
return raw_limit
def _next_jmap_mailbox_cursor(
*,
tenant_id: str,
profile_id: str,
folder: str,
limit: int,
offset: int,
total_count: int,
query_state: str,
messages,
query: str,
) -> tuple[str | None, bool]:
next_offset = offset + len(messages)
if next_offset >= total_count or not messages:
return None, True
return (
encode_keyset_cursor(
JMAP_MAILBOX_MESSAGES_CURSOR_SCOPE,
fingerprint=_jmap_mailbox_cursor_fingerprint(
tenant_id=tenant_id,
profile_id=profile_id,
folder=folder,
limit=limit,
query=query,
),
values={
"offset": next_offset,
"limit": limit,
"query_state": query_state,
},
),
True,
)
def _mailbox_folder_response(
result,
*,
@@ -1209,12 +1324,14 @@ def _mailbox_folder_response(
indexed_at=None,
) -> MailImapFolderListResponse:
folders = [MailImapFolderResponse(name=item.name, flags=item.flags, message_count=item.message_count, unseen_count=item.unseen_count) for item in result.folders]
protocol = getattr(result, "protocol", "imap")
return MailImapFolderListResponse(
ok=True,
protocol=protocol,
host=result.host,
port=result.port,
security=result.security,
message=f"Found {len(folders)} IMAP folder(s).",
message=f"Found {len(folders)} {protocol.upper()} folder(s).",
folders=folders,
detected_sent_folder=result.detected_sent_folder,
detected_folder_mappings=getattr(result, "detected_folder_mappings", None) or {},
@@ -1337,6 +1454,85 @@ def _imap_config_for_principal(
raise MailProfileError(str(exc)) from exc
def _jmap_config_for_principal(
session: Session,
*,
principal: ApiPrincipal,
profile_id: str,
server_id: str | None = None,
credential_id: str | None = None,
campaign_id: str | None = None,
) -> JmapConfig:
profile = _get_profile_for_principal(
session,
principal=principal,
profile_id=profile_id,
require_active=True,
)
try:
resolved = resolve_mail_transport(
session,
profile=profile,
protocol="jmap",
context=_transport_context_for_principal(
session,
principal=principal,
campaign_id=campaign_id,
),
server_id=server_id,
credential_id=credential_id,
)
except MailServerHierarchyError as exc:
raise MailProfileError(str(exc)) from exc
if not isinstance(resolved.config, JmapConfig):
raise MailProfileError("Mail-server profile has no JMAP configuration")
_assert_jmap_policy_allows_config(
session,
principal=principal,
profile=profile,
config=resolved.config,
campaign_id=campaign_id,
)
return resolved.config
def _assert_jmap_policy_allows_config(
session: Session,
*,
principal: ApiPrincipal,
profile,
config: JmapConfig | dict[str, Any],
campaign_id: str | None = None,
) -> None:
session_url = config.session_url if isinstance(config, JmapConfig) else config.get("session_url")
scope_type = profile.scope_type or "tenant"
scope_id = profile.scope_id
policy_context: dict[str, str | None] = {}
if campaign_id or scope_type == "campaign":
policy_context["campaign_id"] = campaign_id or scope_id
elif scope_type == "user":
policy_context["owner_user_id"] = scope_id
elif scope_type == "group":
policy_context["owner_group_id"] = scope_id
assert_mail_policy_allows_transport(
session,
tenant_id=principal.tenant_id,
smtp=None,
jmap={"session_url": session_url},
**policy_context,
)
def _require_mailbox_protocol(value: str) -> str:
clean = value.strip().casefold() if isinstance(value, str) else "imap"
if clean not in {"imap", "jmap"}:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="Mailbox protocol must be imap or jmap",
)
return clean
def _parent_mail_profile_policy_payload(session: Session, *, tenant_id: str, scope_type: str, scope_id: str | None):
if scope_type == "system":
return None
@@ -1625,6 +1821,13 @@ def create_profile_server(
principal=principal,
profile_id=profile_id,
)
if payload.protocol == "jmap":
_assert_jmap_policy_allows_config(
session,
principal=principal,
profile=profile,
config=payload.config,
)
server = create_mail_server_endpoint(
session,
profile=profile,
@@ -1680,6 +1883,13 @@ def update_profile_server(
)
if server.protocol == "pop3":
_require_scope(principal, "mail:pop3:manage")
if server.protocol == "jmap" and payload.config is not None:
_assert_jmap_policy_allows_config(
session,
principal=principal,
profile=profile,
config=payload.config,
)
update_mail_server_endpoint(
session,
server=server,
@@ -1706,7 +1916,7 @@ def update_profile_server(
profile=profile,
server=server,
)
except MailServerHierarchyError as exc:
except (MailProfileError, MailServerHierarchyError) as exc:
session.rollback()
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
@@ -2696,6 +2906,73 @@ def test_profile_imap(
return MailConnectionTestResponse(ok=False, protocol="imap", message=_safe_error_message(exc), details={"error_type": exc.__class__.__name__})
@router.post("/profiles/{profile_id}/test-jmap", response_model=MailConnectionTestResponse)
def test_profile_jmap(
profile_id: str,
server_id: str | None = Query(default=None),
credential_id: str | None = Query(default=None),
campaign_id: str | None = Query(default=None),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_scope(principal, "mail:profile:test")
_require_scope(principal, "mail:profile:use")
try:
jmap = _jmap_config_for_principal(
session,
principal=principal,
profile_id=profile_id,
server_id=server_id,
credential_id=credential_id,
campaign_id=campaign_id,
)
result = test_jmap_connection(jmap_config=jmap)
return MailConnectionTestResponse(
ok=True,
protocol="jmap",
host=result.host,
port=result.port,
security=result.security,
message="JMAP connection successful.",
details={
"authenticated": result.authenticated,
"account_id": result.account_id,
"session_state": result.session_state,
"capabilities": list(result.capabilities),
},
)
except MailProfileError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
except (JmapConfigurationError, JmapCapabilityError) as exc:
return MailConnectionTestResponse(
ok=False,
protocol="jmap",
message=str(exc),
details={"error_type": exc.__class__.__name__},
)
except JmapAuthenticationError as exc:
return MailConnectionTestResponse(
ok=False,
protocol="jmap",
message=str(exc),
details={"error_type": "authentication"},
)
except JmapPermissionError as exc:
return MailConnectionTestResponse(
ok=False,
protocol="jmap",
message=str(exc),
details={"error_type": "permission"},
)
except JmapProviderError as exc:
return MailConnectionTestResponse(
ok=False,
protocol="jmap",
message=str(exc),
details={"error_type": "transient_provider"},
)
def _resolve_profile_pop3_transport(
session: Session,
*,
@@ -3097,11 +3374,22 @@ def list_profile_mailbox_folders(
profile_id: str,
include_status: bool = Query(default=False),
refresh: bool = False,
protocol: str = Query(default="imap"),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_mailbox_read_scope(principal)
mailbox_protocol = _require_mailbox_protocol(protocol)
try:
if mailbox_protocol == "jmap":
jmap = _jmap_config_for_principal(
session,
principal=principal,
profile_id=profile_id,
)
return _mailbox_folder_response(
list_jmap_folders(jmap_config=jmap)
)
imap = _imap_config_for_principal(session, principal=principal, profile_id=profile_id)
if not include_status and not refresh:
cached = cached_mailbox_folders(session, tenant_id=principal.tenant_id, profile_id=profile_id)
@@ -3144,6 +3432,12 @@ def list_profile_mailbox_folders(
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except ImapAppendError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
except (JmapConfigurationError, JmapCapabilityError) as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except (JmapAuthenticationError, JmapPermissionError) as exc:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc
except JmapProviderError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
@router.get("/profiles/{profile_id}/mailbox/bootstrap", response_model=MailMailboxBootstrapResponse)
@@ -3154,11 +3448,56 @@ def bootstrap_profile_mailbox(
offset: int = Query(default=0, ge=0, le=100000),
refresh: bool = False,
include_status: bool = Query(default=False),
protocol: str = Query(default="imap"),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
) -> MailMailboxBootstrapResponse:
_require_mailbox_read_scope(principal)
mailbox_protocol = _require_mailbox_protocol(protocol)
try:
if mailbox_protocol == "jmap":
jmap = _jmap_config_for_principal(
session,
principal=principal,
profile_id=profile_id,
)
result = load_jmap_mailbox_bootstrap(
jmap_config=jmap,
folder=folder,
limit=limit,
offset=offset,
)
next_cursor, cursor_stable = _next_jmap_mailbox_cursor(
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=result.messages.folder,
limit=result.messages.limit,
offset=result.messages.offset,
query_state=result.messages.uidvalidity,
messages=result.messages.messages,
total_count=result.messages.total_count,
query="",
)
return MailMailboxBootstrapResponse(
profile_id=profile_id,
folder=result.messages.folder,
folders=_mailbox_folder_response(result.folders),
messages=_mailbox_messages_response(
profile_id=profile_id,
folder=result.messages.folder,
host=result.messages.host,
port=result.messages.port,
security=result.messages.security,
total_count=result.messages.total_count,
offset=result.messages.offset,
limit=result.messages.limit,
cursor=None,
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=False,
messages=result.messages.messages,
),
)
imap = _imap_config_for_principal(session, principal=principal, profile_id=profile_id)
if not refresh:
cached_folders = cached_mailbox_folders(session, tenant_id=principal.tenant_id, profile_id=profile_id)
@@ -3285,6 +3624,12 @@ def bootstrap_profile_mailbox(
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except ImapAppendError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
except (JmapConfigurationError, JmapCapabilityError) as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except (JmapAuthenticationError, JmapPermissionError) as exc:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc
except JmapProviderError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
@router.get("/profiles/{profile_id}/mailbox/messages", response_model=MailMailboxMessageListResponse)
@@ -3295,11 +3640,68 @@ def list_profile_mailbox_messages(
offset: int = Query(default=0, ge=0, le=100000),
cursor: str | None = None,
refresh: bool = False,
protocol: str = Query(default="imap"),
q: str | None = Query(default=None, max_length=500),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_mailbox_read_scope(principal)
mailbox_protocol = _require_mailbox_protocol(protocol)
try:
if mailbox_protocol == "jmap":
clean_query = q.strip() if isinstance(q, str) else ""
effective_limit = _jmap_mailbox_cursor_limit(cursor, limit)
fingerprint = _jmap_mailbox_cursor_fingerprint(
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=folder,
limit=effective_limit,
query=clean_query,
)
effective_offset, expected_query_state = _jmap_mailbox_cursor_position(
cursor,
fingerprint=fingerprint,
offset=offset,
)
jmap = _jmap_config_for_principal(
session,
principal=principal,
profile_id=profile_id,
)
result = list_jmap_messages(
jmap_config=jmap,
folder=folder,
limit=effective_limit,
offset=effective_offset,
expected_query_state=expected_query_state,
query=clean_query,
)
next_cursor, cursor_stable = _next_jmap_mailbox_cursor(
tenant_id=principal.tenant_id,
profile_id=profile_id,
folder=result.folder,
limit=result.limit,
offset=result.offset,
total_count=result.total_count,
query_state=result.uidvalidity,
messages=result.messages,
query=clean_query,
)
return _mailbox_messages_response(
profile_id=profile_id,
folder=result.folder,
host=result.host,
port=result.port,
security=result.security,
total_count=result.total_count,
offset=result.offset,
limit=result.limit,
cursor=cursor,
next_cursor=next_cursor,
cursor_stable=cursor_stable,
full=result.cursor_reset,
messages=result.messages,
)
imap = _imap_config_for_principal(session, principal=principal, profile_id=profile_id)
effective_limit = _mailbox_cursor_limit(cursor, limit)
fingerprint = _mailbox_cursor_fingerprint(tenant_id=principal.tenant_id, profile_id=profile_id, folder=folder, limit=effective_limit)
@@ -3407,6 +3809,55 @@ def list_profile_mailbox_messages(
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except ImapAppendError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
except (JmapConfigurationError, JmapCapabilityError) as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except (JmapAuthenticationError, JmapPermissionError) as exc:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc
except JmapProviderError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
@router.get(
"/profiles/{profile_id}/mailbox/changes",
response_model=MailMailboxChangesResponse,
)
def get_profile_mailbox_changes(
profile_id: str,
since_state: str = Query(min_length=1, max_length=1_000),
max_changes: int = Query(default=500, ge=1, le=1_000),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_mailbox_read_scope(principal)
try:
jmap = _jmap_config_for_principal(
session,
principal=principal,
profile_id=profile_id,
)
result = get_jmap_email_changes(
jmap_config=jmap,
since_state=since_state,
max_changes=max_changes,
)
return MailMailboxChangesResponse(
profile_id=profile_id,
account_id=result.account_id,
old_state=result.old_state,
new_state=result.new_state,
has_more_changes=result.has_more_changes,
created=list(result.created),
updated=list(result.updated),
destroyed=list(result.destroyed),
)
except MailProfileError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
except (JmapConfigurationError, JmapCapabilityError) as exc:
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc
except (JmapAuthenticationError, JmapPermissionError) as exc:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc
except JmapProviderError as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
@router.get("/profiles/{profile_id}/mailbox/messages/{message_uid}", response_model=MailMailboxMessageResponse)
@@ -3414,11 +3865,32 @@ def get_profile_mailbox_message(
profile_id: str,
message_uid: str,
folder: str = Query(default="INBOX", min_length=1, max_length=255),
protocol: str = Query(default="imap"),
principal: ApiPrincipal = Depends(get_api_principal),
session: Session = Depends(get_session),
):
_require_mailbox_read_scope(principal)
mailbox_protocol = _require_mailbox_protocol(protocol)
try:
if mailbox_protocol == "jmap":
jmap = _jmap_config_for_principal(
session,
principal=principal,
profile_id=profile_id,
)
result = get_jmap_message(
jmap_config=jmap,
folder=folder,
email_id=message_uid,
)
return MailMailboxMessageResponse(
profile_id=profile_id,
folder=result.folder,
host=result.host,
port=result.port,
security=result.security,
message=_mailbox_detail_response(result.message),
)
imap = _imap_config_for_principal(session, principal=principal, profile_id=profile_id)
result = get_imap_message(imap_config=imap, folder=folder, uid=message_uid)
return MailMailboxMessageResponse(
@@ -3436,6 +3908,13 @@ def get_profile_mailbox_message(
except ImapAppendError as exc:
status_code = status.HTTP_404_NOT_FOUND if "not found" in str(exc).casefold() else status.HTTP_502_BAD_GATEWAY
raise HTTPException(status_code=status_code, detail=str(exc)) from exc
except (JmapConfigurationError, JmapCapabilityError) as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except (JmapAuthenticationError, JmapPermissionError) as exc:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) from exc
except JmapProviderError as exc:
status_code = status.HTTP_404_NOT_FOUND if "not found" in str(exc).casefold() else status.HTTP_502_BAD_GATEWAY
raise HTTPException(status_code=status_code, detail=str(exc)) from exc
def _safe_error_message(exc: Exception) -> str: