feat(mail): add governed POP3 legacy import
Module Package Release / publish-packages (push) Successful in 11s
Module Package Release / publish-packages (push) Successful in 11s
This commit is contained in:
@@ -0,0 +1,492 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import poplib
|
||||
import socket
|
||||
import ssl
|
||||
from dataclasses import dataclass
|
||||
from email import policy
|
||||
from email.message import Message
|
||||
from email.parser import BytesParser
|
||||
from typing import Iterable
|
||||
|
||||
from govoplan_core.security.outbound_http import (
|
||||
OutboundHttpError,
|
||||
create_outbound_connection,
|
||||
validate_outbound_host,
|
||||
)
|
||||
from govoplan_mail.backend.config import Pop3Config, TransportSecurity
|
||||
|
||||
|
||||
class _OutboundPolicyPOP3(poplib.POP3):
|
||||
def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def]
|
||||
return create_outbound_connection(
|
||||
self.host,
|
||||
self.port,
|
||||
timeout=timeout,
|
||||
label="POP3 legacy import",
|
||||
)
|
||||
|
||||
|
||||
class _OutboundPolicyPOP3SSL(poplib.POP3_SSL):
|
||||
def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def]
|
||||
sock = create_outbound_connection(
|
||||
self.host,
|
||||
self.port,
|
||||
timeout=timeout,
|
||||
label="POP3 legacy import",
|
||||
)
|
||||
try:
|
||||
return self.context.wrap_socket(sock, server_hostname=self.host)
|
||||
except Exception:
|
||||
sock.close()
|
||||
raise
|
||||
|
||||
|
||||
class Pop3ConfigurationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class Pop3ProviderError(RuntimeError):
|
||||
def __init__(self, message: str, *, outcome_unknown: bool = False):
|
||||
super().__init__(message)
|
||||
self.outcome_unknown = outcome_unknown
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pop3LoginTestResult:
|
||||
host: str
|
||||
port: int
|
||||
security: str
|
||||
authenticated: bool
|
||||
message_count: int
|
||||
mailbox_size_bytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pop3MessageSummary:
|
||||
message_number: int
|
||||
uidl: str
|
||||
subject: str | None
|
||||
from_header: str | None
|
||||
to_header: str | None
|
||||
date: str | None
|
||||
message_id: str | None
|
||||
size_bytes: int
|
||||
body_preview: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pop3PreviewResult:
|
||||
host: str
|
||||
port: int
|
||||
security: str
|
||||
message_count: int
|
||||
mailbox_size_bytes: int
|
||||
messages: tuple[Pop3MessageSummary, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pop3DownloadedMessage:
|
||||
message_number: int
|
||||
uidl: str
|
||||
raw: bytes
|
||||
raw_sha256: str
|
||||
summary: Pop3MessageSummary
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pop3DeletionResult:
|
||||
deleted_uidls: tuple[str, ...]
|
||||
|
||||
|
||||
def _require_pop3_config(config: Pop3Config) -> tuple[str, int]:
|
||||
if not config.legacy_import_enabled:
|
||||
raise Pop3ConfigurationError(
|
||||
"POP3 legacy import is disabled for the selected server"
|
||||
)
|
||||
if not config.host:
|
||||
raise Pop3ConfigurationError("POP3 host is required")
|
||||
if not config.port:
|
||||
raise Pop3ConfigurationError("POP3 port is required")
|
||||
if not config.username or not config.password:
|
||||
raise Pop3ConfigurationError("POP3 username and password are required")
|
||||
return config.host, config.port
|
||||
|
||||
|
||||
def _open_pop3(config: Pop3Config) -> poplib.POP3:
|
||||
host, port = _require_pop3_config(config)
|
||||
try:
|
||||
validate_outbound_host(host, port=port, label="POP3 legacy import")
|
||||
except OutboundHttpError as exc:
|
||||
raise Pop3ConfigurationError(str(exc)) from exc
|
||||
|
||||
context = ssl.create_default_context()
|
||||
client: poplib.POP3 | None = None
|
||||
try:
|
||||
if config.security == TransportSecurity.TLS:
|
||||
client = _OutboundPolicyPOP3SSL(
|
||||
host=host,
|
||||
port=port,
|
||||
timeout=config.timeout_seconds,
|
||||
context=context,
|
||||
)
|
||||
else:
|
||||
client = _OutboundPolicyPOP3(
|
||||
host=host,
|
||||
port=port,
|
||||
timeout=config.timeout_seconds,
|
||||
)
|
||||
if config.security == TransportSecurity.STARTTLS:
|
||||
client.stls(context=context)
|
||||
client.user(config.username)
|
||||
client.pass_(config.password)
|
||||
return client
|
||||
except ssl.SSLError as exc:
|
||||
_close_without_commit(client)
|
||||
raise Pop3ProviderError("POP3 TLS negotiation failed") from exc
|
||||
except poplib.error_proto as exc:
|
||||
_close_without_commit(client)
|
||||
raise Pop3ProviderError("POP3 authentication failed") from exc
|
||||
except (OSError, socket.error) as exc:
|
||||
_close_without_commit(client)
|
||||
raise Pop3ProviderError("POP3 connection failed") from exc
|
||||
except Exception:
|
||||
_close_without_commit(client)
|
||||
raise
|
||||
|
||||
|
||||
def test_pop3_login(*, pop3_config: Pop3Config) -> Pop3LoginTestResult:
|
||||
client = _open_pop3(pop3_config)
|
||||
try:
|
||||
message_count, mailbox_size = client.stat()
|
||||
return Pop3LoginTestResult(
|
||||
host=str(pop3_config.host),
|
||||
port=int(pop3_config.port or 0),
|
||||
security=pop3_config.security.value,
|
||||
authenticated=True,
|
||||
message_count=int(message_count),
|
||||
mailbox_size_bytes=int(mailbox_size),
|
||||
)
|
||||
except poplib.error_proto as exc:
|
||||
raise Pop3ProviderError("POP3 mailbox statistics are unavailable") from exc
|
||||
finally:
|
||||
_quit_without_deletions(client)
|
||||
|
||||
|
||||
def preview_pop3_messages(
|
||||
*,
|
||||
pop3_config: Pop3Config,
|
||||
limit: int = 50,
|
||||
) -> Pop3PreviewResult:
|
||||
clean_limit = max(1, min(int(limit), 100))
|
||||
client = _open_pop3(pop3_config)
|
||||
try:
|
||||
message_count, mailbox_size = client.stat()
|
||||
uidls = _uidl_map(client)
|
||||
sizes = _size_map(client)
|
||||
selected_numbers = sorted(uidls, reverse=True)[:clean_limit]
|
||||
messages = tuple(
|
||||
_preview_message(
|
||||
client,
|
||||
message_number=number,
|
||||
uidl=uidls[number],
|
||||
size_bytes=sizes.get(number, 0),
|
||||
body_lines=pop3_config.preview_body_lines,
|
||||
max_message_bytes=pop3_config.max_message_bytes,
|
||||
)
|
||||
for number in selected_numbers
|
||||
)
|
||||
return Pop3PreviewResult(
|
||||
host=str(pop3_config.host),
|
||||
port=int(pop3_config.port or 0),
|
||||
security=pop3_config.security.value,
|
||||
message_count=int(message_count),
|
||||
mailbox_size_bytes=int(mailbox_size),
|
||||
messages=messages,
|
||||
)
|
||||
except poplib.error_proto as exc:
|
||||
raise Pop3ProviderError("POP3 message preview failed") from exc
|
||||
finally:
|
||||
_quit_without_deletions(client)
|
||||
|
||||
|
||||
def download_pop3_messages(
|
||||
*,
|
||||
pop3_config: Pop3Config,
|
||||
uidls: Iterable[str],
|
||||
) -> tuple[Pop3DownloadedMessage, ...]:
|
||||
selected_uidls = tuple(dict.fromkeys(_required_uidl(value) for value in uidls))
|
||||
if not selected_uidls:
|
||||
raise Pop3ConfigurationError("Select at least one POP3 message to import")
|
||||
if len(selected_uidls) > 100:
|
||||
raise Pop3ConfigurationError("At most 100 POP3 messages can be imported at once")
|
||||
|
||||
client = _open_pop3(pop3_config)
|
||||
try:
|
||||
uidl_by_number = _uidl_map(client)
|
||||
number_by_uidl = {uidl: number for number, uidl in uidl_by_number.items()}
|
||||
missing = [uidl for uidl in selected_uidls if uidl not in number_by_uidl]
|
||||
if missing:
|
||||
raise Pop3ProviderError(
|
||||
"One or more previewed POP3 messages are no longer available; refresh the preview"
|
||||
)
|
||||
sizes = _size_map(client)
|
||||
advertised_batch_size = sum(
|
||||
max(0, int(sizes.get(number_by_uidl[uidl], 0)))
|
||||
for uidl in selected_uidls
|
||||
)
|
||||
if advertised_batch_size > pop3_config.max_batch_bytes:
|
||||
raise Pop3ProviderError(
|
||||
"The selected POP3 messages exceed the configured batch size limit"
|
||||
)
|
||||
downloaded: list[Pop3DownloadedMessage] = []
|
||||
downloaded_bytes = 0
|
||||
for uidl in selected_uidls:
|
||||
number = number_by_uidl[uidl]
|
||||
advertised_size = sizes.get(number, 0)
|
||||
if advertised_size > pop3_config.max_message_bytes:
|
||||
raise Pop3ProviderError(
|
||||
f"POP3 message {uidl} exceeds the configured import size limit"
|
||||
)
|
||||
_response, lines, _octets = client.retr(number)
|
||||
raw = _message_bytes(lines)
|
||||
if len(raw) > pop3_config.max_message_bytes:
|
||||
raise Pop3ProviderError(
|
||||
f"POP3 message {uidl} exceeds the configured import size limit"
|
||||
)
|
||||
downloaded_bytes += len(raw)
|
||||
if downloaded_bytes > pop3_config.max_batch_bytes:
|
||||
raise Pop3ProviderError(
|
||||
"The selected POP3 messages exceed the configured batch size limit"
|
||||
)
|
||||
summary = _message_summary(
|
||||
raw,
|
||||
message_number=number,
|
||||
uidl=uidl,
|
||||
size_bytes=len(raw),
|
||||
)
|
||||
downloaded.append(
|
||||
Pop3DownloadedMessage(
|
||||
message_number=number,
|
||||
uidl=uidl,
|
||||
raw=raw,
|
||||
raw_sha256=hashlib.sha256(raw).hexdigest(),
|
||||
summary=summary,
|
||||
)
|
||||
)
|
||||
return tuple(downloaded)
|
||||
except poplib.error_proto as exc:
|
||||
raise Pop3ProviderError("POP3 message download failed") from exc
|
||||
finally:
|
||||
_quit_without_deletions(client)
|
||||
|
||||
|
||||
def delete_pop3_messages(
|
||||
*,
|
||||
pop3_config: Pop3Config,
|
||||
uidls: Iterable[str],
|
||||
) -> Pop3DeletionResult:
|
||||
selected_uidls = tuple(dict.fromkeys(_required_uidl(value) for value in uidls))
|
||||
if not selected_uidls:
|
||||
return Pop3DeletionResult(deleted_uidls=())
|
||||
if not pop3_config.allow_delete_after_import:
|
||||
raise Pop3ConfigurationError(
|
||||
"POP3 delete-after-import is disabled for the selected server"
|
||||
)
|
||||
|
||||
client = _open_pop3(pop3_config)
|
||||
quit_started = False
|
||||
try:
|
||||
number_by_uidl = {
|
||||
uidl: number for number, uidl in _uidl_map(client).items()
|
||||
}
|
||||
missing = [uidl for uidl in selected_uidls if uidl not in number_by_uidl]
|
||||
if missing:
|
||||
raise Pop3ProviderError(
|
||||
"One or more imported POP3 messages are no longer available for deletion"
|
||||
)
|
||||
for uidl in selected_uidls:
|
||||
client.dele(number_by_uidl[uidl])
|
||||
quit_started = True
|
||||
client.quit()
|
||||
return Pop3DeletionResult(deleted_uidls=selected_uidls)
|
||||
except Pop3ProviderError:
|
||||
_close_without_commit(client)
|
||||
raise
|
||||
except poplib.error_proto as exc:
|
||||
_close_without_commit(client)
|
||||
raise Pop3ProviderError(
|
||||
"POP3 deletion outcome is unknown" if quit_started else "POP3 deletion was rejected",
|
||||
outcome_unknown=quit_started,
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
_close_without_commit(client)
|
||||
raise Pop3ProviderError(
|
||||
"POP3 deletion outcome is unknown" if quit_started else "POP3 deletion failed",
|
||||
outcome_unknown=quit_started,
|
||||
) from exc
|
||||
|
||||
|
||||
def _uidl_map(client: poplib.POP3) -> dict[int, str]:
|
||||
_response, lines, _octets = client.uidl()
|
||||
result: dict[int, str] = {}
|
||||
for raw_line in lines:
|
||||
parts = bytes(raw_line).decode("utf-8", errors="replace").split(maxsplit=1)
|
||||
if len(parts) != 2 or not parts[0].isdigit():
|
||||
continue
|
||||
uidl = _required_uidl(parts[1])
|
||||
result[int(parts[0])] = uidl
|
||||
if not result:
|
||||
raise Pop3ProviderError(
|
||||
"The POP3 server does not provide stable UIDL identifiers; safe import is unavailable"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _size_map(client: poplib.POP3) -> dict[int, int]:
|
||||
_response, lines, _octets = client.list()
|
||||
result: dict[int, int] = {}
|
||||
for raw_line in lines:
|
||||
parts = bytes(raw_line).decode("ascii", errors="ignore").split(maxsplit=1)
|
||||
if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit():
|
||||
result[int(parts[0])] = int(parts[1])
|
||||
return result
|
||||
|
||||
|
||||
def _preview_message(
|
||||
client: poplib.POP3,
|
||||
*,
|
||||
message_number: int,
|
||||
uidl: str,
|
||||
size_bytes: int,
|
||||
body_lines: int,
|
||||
max_message_bytes: int,
|
||||
) -> Pop3MessageSummary:
|
||||
raw: bytes | None = None
|
||||
try:
|
||||
_response, lines, _octets = client.top(message_number, body_lines)
|
||||
raw = _message_bytes(lines)
|
||||
except (poplib.error_proto, AttributeError):
|
||||
# TOP is optional. Never use RETR as a preview fallback when the
|
||||
# advertised message already exceeds the configured download bound.
|
||||
if size_bytes > max_message_bytes:
|
||||
return Pop3MessageSummary(
|
||||
message_number=message_number,
|
||||
uidl=uidl,
|
||||
subject=None,
|
||||
from_header=None,
|
||||
to_header=None,
|
||||
date=None,
|
||||
message_id=None,
|
||||
size_bytes=size_bytes,
|
||||
body_preview=None,
|
||||
)
|
||||
_response, lines, _octets = client.retr(message_number)
|
||||
raw = _message_bytes(lines)
|
||||
if len(raw) > min(max_message_bytes, 256 * 1024):
|
||||
return Pop3MessageSummary(
|
||||
message_number=message_number,
|
||||
uidl=uidl,
|
||||
subject=None,
|
||||
from_header=None,
|
||||
to_header=None,
|
||||
date=None,
|
||||
message_id=None,
|
||||
size_bytes=size_bytes,
|
||||
body_preview=None,
|
||||
)
|
||||
return _message_summary(
|
||||
raw,
|
||||
message_number=message_number,
|
||||
uidl=uidl,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _message_summary(
|
||||
raw: bytes,
|
||||
*,
|
||||
message_number: int,
|
||||
uidl: str,
|
||||
size_bytes: int,
|
||||
) -> Pop3MessageSummary:
|
||||
message = BytesParser(policy=policy.default).parsebytes(raw)
|
||||
return Pop3MessageSummary(
|
||||
message_number=message_number,
|
||||
uidl=uidl,
|
||||
subject=_header(message, "Subject"),
|
||||
from_header=_header(message, "From"),
|
||||
to_header=_header(message, "To"),
|
||||
date=_header(message, "Date"),
|
||||
message_id=_header(message, "Message-ID"),
|
||||
size_bytes=max(0, int(size_bytes)),
|
||||
body_preview=_body_preview(message),
|
||||
)
|
||||
|
||||
|
||||
def _header(message: Message, name: str) -> str | None:
|
||||
value = message.get(name)
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text[:2_000] or None
|
||||
|
||||
|
||||
def _body_preview(message: Message) -> str | None:
|
||||
body = message.get_body(preferencelist=("plain",)) if message.is_multipart() else message
|
||||
if body is None:
|
||||
return None
|
||||
try:
|
||||
text = body.get_content()
|
||||
except Exception:
|
||||
payload = body.get_payload(decode=True)
|
||||
text = payload.decode("utf-8", errors="replace") if isinstance(payload, bytes) else str(payload or "")
|
||||
normalized = " ".join(str(text).split())
|
||||
return normalized[:500] or None
|
||||
|
||||
|
||||
def _message_bytes(lines: Iterable[bytes]) -> bytes:
|
||||
return b"\r\n".join(bytes(line) for line in lines) + b"\r\n"
|
||||
|
||||
|
||||
def _required_uidl(value: object) -> str:
|
||||
clean = str(value or "").strip()
|
||||
if not clean or len(clean) > 500 or any(char.isspace() for char in clean):
|
||||
raise Pop3ConfigurationError("POP3 UIDL must be a non-empty token")
|
||||
return clean
|
||||
|
||||
|
||||
def _quit_without_deletions(client: poplib.POP3) -> None:
|
||||
try:
|
||||
client.quit()
|
||||
except Exception:
|
||||
_close_without_commit(client)
|
||||
|
||||
|
||||
def _close_without_commit(client: poplib.POP3 | None) -> None:
|
||||
if client is None:
|
||||
return
|
||||
try:
|
||||
client.rset()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Pop3ConfigurationError",
|
||||
"Pop3DeletionResult",
|
||||
"Pop3DownloadedMessage",
|
||||
"Pop3LoginTestResult",
|
||||
"Pop3MessageSummary",
|
||||
"Pop3PreviewResult",
|
||||
"Pop3ProviderError",
|
||||
"delete_pop3_messages",
|
||||
"download_pop3_messages",
|
||||
"preview_pop3_messages",
|
||||
"test_pop3_login",
|
||||
]
|
||||
Reference in New Issue
Block a user