diff --git a/pyproject.toml b/pyproject.toml index 9946b91..22220dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ requires-python = ">=3.12" license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ - "govoplan-core>=0.1.8", + "govoplan-core>=0.1.9", "pydantic>=2,<3", "redis>=5,<6", "SQLAlchemy>=2,<3", diff --git a/src/govoplan_mail/backend/sending/imap.py b/src/govoplan_mail/backend/sending/imap.py index dc6cb08..b742f2e 100644 --- a/src/govoplan_mail/backend/sending/imap.py +++ b/src/govoplan_mail/backend/sending/imap.py @@ -12,6 +12,13 @@ from email.message import EmailMessage from email.parser import BytesParser from typing import Any +from govoplan_core.security.outbound_http import ( + OutboundHttpError, + create_outbound_connection, + response_limit, + validate_outbound_host, +) + from govoplan_mail.backend.config import ImapConfig, TransportSecurity from govoplan_mail.backend.dev.mock_mailbox import ( MOCK_IMAP_FOLDERS, @@ -25,6 +32,31 @@ from govoplan_mail.backend.dev.mock_mailbox import ( logger = logging.getLogger(__name__) +class _OutboundPolicyIMAP4(imaplib.IMAP4): + def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def] + return create_outbound_connection( + self.host, + self.port, + timeout=timeout, + label="IMAP connector", + ) + + +class _OutboundPolicyIMAP4SSL(imaplib.IMAP4_SSL): + def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def] + sock = create_outbound_connection( + self.host, + self.port, + timeout=timeout, + label="IMAP connector", + ) + try: + return self.ssl_context.wrap_socket(sock, server_hostname=self.host) + except Exception: + sock.close() + raise + + class ImapConfigurationError(ValueError): """Raised when IMAP settings are incomplete or inconsistent.""" @@ -163,13 +195,22 @@ def _require_imap_config(config: ImapConfig) -> tuple[str, int]: def _open_imap(config: ImapConfig) -> imaplib.IMAP4: host, port = _require_imap_config(config) + try: + validate_outbound_host(host, port=port, label="IMAP connector") + except OutboundHttpError as exc: + raise ImapConfigurationError(str(exc)) from exc context = ssl.create_default_context() try: if config.security == TransportSecurity.TLS: - client: imaplib.IMAP4 = imaplib.IMAP4_SSL(host=host, port=port, timeout=config.timeout_seconds, ssl_context=context) + client: imaplib.IMAP4 = _OutboundPolicyIMAP4SSL( + host=host, + port=port, + timeout=config.timeout_seconds, + ssl_context=context, + ) else: - client = imaplib.IMAP4(host=host, port=port, timeout=config.timeout_seconds) + client = _OutboundPolicyIMAP4(host=host, port=port, timeout=config.timeout_seconds) if config.security == TransportSecurity.STARTTLS: typ, data = client.starttls(ssl_context=context) if typ != "OK": @@ -704,18 +745,22 @@ def _select_readonly(client: imaplib.IMAP4, folder: str) -> tuple[int, str | Non def _fetch_message_by_uid(client: imaplib.IMAP4, uid: str) -> tuple[str, list[str], int | None, bytes]: - typ, data = client.uid("fetch", str(uid), "(UID FLAGS RFC822.SIZE BODY.PEEK[])") + max_bytes = response_limit("file") + typ, data = client.uid("fetch", str(uid), f"(UID FLAGS RFC822.SIZE BODY.PEEK[]<0.{max_bytes + 1}>)") if typ != "OK": raise ImapAppendError(f"IMAP message {uid!r} fetch failed: {data!r}", temporary=True) fetched_uid, flags, size_bytes, raw = _parse_fetch_response(data) + _ensure_imap_payload_within_limit(raw, declared_size=size_bytes, max_bytes=max_bytes, label="IMAP message") return fetched_uid or str(uid), flags, size_bytes, raw def _fetch_message_by_sequence(client: imaplib.IMAP4, sequence: str) -> tuple[str, list[str], int | None, bytes]: - typ, data = client.fetch(str(sequence), "(UID FLAGS RFC822.SIZE BODY.PEEK[])") + max_bytes = response_limit("file") + typ, data = client.fetch(str(sequence), f"(UID FLAGS RFC822.SIZE BODY.PEEK[]<0.{max_bytes + 1}>)") if typ != "OK": raise ImapAppendError(f"IMAP message sequence {sequence!r} fetch failed: {data!r}", temporary=True) fetched_uid, flags, size_bytes, raw = _parse_fetch_response(data) + _ensure_imap_payload_within_limit(raw, declared_size=size_bytes, max_bytes=max_bytes, label="IMAP message") if not fetched_uid: raise ImapAppendError(f"IMAP message sequence {sequence!r} fetch returned no UID", temporary=True) return fetched_uid, flags, size_bytes, raw @@ -734,12 +779,19 @@ def _sequence_set(sequences: list[str]) -> str: def _fetch_message_summaries_by_sequence(client: imaplib.IMAP4, sequences: list[str], folder: str) -> list[ImapMailboxMessageSummary]: if not sequences: return [] - typ, data = client.fetch(_sequence_set(sequences), "(UID FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (SUBJECT FROM TO CC DATE MESSAGE-ID)])") + max_bytes = response_limit("structured") + per_message_bytes = max(1, max_bytes // len(sequences)) + 1 + typ, data = client.fetch( + _sequence_set(sequences), + f"(UID FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (SUBJECT FROM TO CC DATE MESSAGE-ID)]<0.{per_message_bytes}>)", + ) if typ != "OK": raise ImapAppendError(f"IMAP message summary fetch failed: {data!r}", temporary=True) by_sequence: dict[str, ImapMailboxMessageSummary] = {} - for sequence, fetched_uid, flags, size_bytes, headers in _parse_fetch_parts_with_sequence(data): + parsed_parts = _parse_fetch_parts_with_sequence(data) + _ensure_imap_batch_within_limit(parsed_parts, max_bytes=max_bytes, label="IMAP message summary response") + for sequence, fetched_uid, flags, size_bytes, headers in parsed_parts: if not sequence or not fetched_uid: continue by_sequence[sequence] = _message_summary_from_headers(uid=fetched_uid, folder=folder, headers=headers, flags=flags, size_bytes=size_bytes) @@ -791,12 +843,20 @@ def _paged_descending_uids( def _fetch_message_summaries_by_uid(client: imaplib.IMAP4, uids: list[str], folder: str) -> list[ImapMailboxMessageSummary]: if not uids: return [] - typ, data = client.uid("fetch", _sequence_set(uids), "(UID FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (SUBJECT FROM TO CC DATE MESSAGE-ID)])") + max_bytes = response_limit("structured") + per_message_bytes = max(1, max_bytes // len(uids)) + 1 + typ, data = client.uid( + "fetch", + _sequence_set(uids), + f"(UID FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (SUBJECT FROM TO CC DATE MESSAGE-ID)]<0.{per_message_bytes}>)", + ) if typ != "OK": raise ImapAppendError(f"IMAP message summary fetch failed: {data!r}", temporary=True) by_uid: dict[str, ImapMailboxMessageSummary] = {} - for _sequence, fetched_uid, flags, size_bytes, headers in _parse_fetch_parts_with_sequence(data): + parsed_parts = _parse_fetch_parts_with_sequence(data) + _ensure_imap_batch_within_limit(parsed_parts, max_bytes=max_bytes, label="IMAP message summary response") + for _sequence, fetched_uid, flags, size_bytes, headers in parsed_parts: if not fetched_uid: continue by_uid[fetched_uid] = _message_summary_from_headers(uid=fetched_uid, folder=folder, headers=headers, flags=flags, size_bytes=size_bytes) @@ -812,6 +872,27 @@ def _fetch_message_summaries_by_uid(client: imaplib.IMAP4, uids: list[str], fold return summaries +def _ensure_imap_payload_within_limit( + payload: bytes, + *, + declared_size: int | None, + max_bytes: int, + label: str, +) -> None: + if (declared_size is not None and declared_size > max_bytes) or len(payload) > max_bytes: + raise ImapAppendError(f"{label} exceeds the deployment limit of {max_bytes} bytes", temporary=False) + + +def _ensure_imap_batch_within_limit( + parts: list[tuple[str | None, str | None, list[str], int | None, bytes]], + *, + max_bytes: int, + label: str, +) -> None: + if sum(len(part[4]) for part in parts) > max_bytes: + raise ImapAppendError(f"{label} exceeds the deployment limit of {max_bytes} bytes", temporary=False) + + def _mock_record_folder(record: dict[str, Any]) -> str: folder = str(record.get("folder") or "").strip() if folder: diff --git a/src/govoplan_mail/backend/sending/smtp.py b/src/govoplan_mail/backend/sending/smtp.py index 0fb158e..f0f1f55 100644 --- a/src/govoplan_mail/backend/sending/smtp.py +++ b/src/govoplan_mail/backend/sending/smtp.py @@ -8,6 +8,12 @@ from dataclasses import dataclass from email.message import EmailMessage from email.utils import formataddr +from govoplan_core.security.outbound_http import ( + OutboundHttpError, + create_outbound_connection, + validate_outbound_host, +) + from govoplan_mail.backend.config import SmtpConfig, TransportSecurity from govoplan_mail.backend.dev.mock_mailbox import ( consume_fail_next_smtp, @@ -19,6 +25,33 @@ from govoplan_mail.backend.dev.mock_mailbox import ( logger = logging.getLogger(__name__) +class _OutboundPolicySMTP(smtplib.SMTP): + def _get_socket(self, host: str, port: int, timeout: float | None): # type: ignore[no-untyped-def] + return create_outbound_connection( + host, + port, + timeout=timeout, + source_address=self.source_address, + label="SMTP connector", + ) + + +class _OutboundPolicySMTPSSL(smtplib.SMTP_SSL): + def _get_socket(self, host: str, port: int, timeout: float | None): # type: ignore[no-untyped-def] + sock = create_outbound_connection( + host, + port, + timeout=timeout, + source_address=self.source_address, + label="SMTP connector", + ) + try: + return self.context.wrap_socket(sock, server_hostname=self._host) + except Exception: + sock.close() + raise + + class SmtpConfigurationError(ValueError): """Raised when SMTP settings are incomplete or inconsistent.""" @@ -75,14 +108,23 @@ def _require_smtp_config(config: SmtpConfig) -> tuple[str, int]: def _open_smtp(config: SmtpConfig) -> smtplib.SMTP: host, port = _require_smtp_config(config) + try: + validate_outbound_host(host, port=port, label="SMTP connector") + except OutboundHttpError as exc: + raise SmtpConfigurationError(str(exc)) from exc context = ssl.create_default_context() try: if config.security == TransportSecurity.TLS: - smtp: smtplib.SMTP = smtplib.SMTP_SSL(host=host, port=port, timeout=config.timeout_seconds, context=context) + smtp: smtplib.SMTP = _OutboundPolicySMTPSSL( + host=host, + port=port, + timeout=config.timeout_seconds, + context=context, + ) smtp.ehlo() else: - smtp = smtplib.SMTP(host=host, port=port, timeout=config.timeout_seconds) + smtp = _OutboundPolicySMTP(host=host, port=port, timeout=config.timeout_seconds) smtp.ehlo() if config.security == TransportSecurity.STARTTLS: smtp.starttls(context=context) diff --git a/tests/test_imap_parser.py b/tests/test_imap_parser.py index 7c1f849..15e78d1 100644 --- a/tests/test_imap_parser.py +++ b/tests/test_imap_parser.py @@ -3,11 +3,16 @@ from __future__ import annotations import unittest from unittest.mock import patch +from govoplan_core.security.outbound_http import OutboundHttpBlocked from govoplan_mail.backend.config import ImapConfig from govoplan_mail.backend.sending.imap import ( + ImapAppendError, + ImapConfigurationError, _detect_sent_folder, _extract_mailbox_name, + _fetch_message_by_uid, _normalize_mailbox_page, + _open_imap, _paged_descending_sequences, _parse_fetch_sequence, _select_readonly, @@ -17,6 +22,31 @@ from govoplan_mail.backend.sending.imap import ( class ImapFolderParserTests(unittest.TestCase): + def test_real_imap_connections_honor_deployment_egress_policy(self): + config = ImapConfig(host="imap.internal", port=993, security="tls") + with patch( + "govoplan_mail.backend.sending.imap.validate_outbound_host", + side_effect=OutboundHttpBlocked("private network blocked"), + ), self.assertRaisesRegex(ImapConfigurationError, "private network blocked"): + _open_imap(config) + + def test_imap_revalidates_and_pins_at_connection_time(self): + config = ImapConfig(host="imap.example.test", port=993, security="tls") + public = [(2, 1, 6, "", ("93.184.216.34", 993))] + private = [(2, 1, 6, "", ("127.0.0.1", 993))] + with patch.dict( + "os.environ", + {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"}, + ), patch( + "govoplan_core.security.outbound_http.socket.getaddrinfo", + side_effect=(public, private), + ), patch("govoplan_core.security.outbound_http.socket.socket") as socket_factory, self.assertRaisesRegex( + OutboundHttpBlocked, + "non-public network", + ): + _open_imap(config) + socket_factory.assert_not_called() + def test_extracts_quoted_mailbox_after_quoted_delimiter(self): self.assertEqual( _extract_mailbox_name(b'(\\HasNoChildren \\Sent) "/" "Sent Items"'), @@ -49,6 +79,23 @@ class ImapFolderParserTests(unittest.TestCase): class ImapMessagePaginationTests(unittest.TestCase): + def test_full_message_fetch_uses_partial_range_and_deployment_limit(self): + class Client: + command = "" + + def uid(self, command, uid, fetch_spec): + del command, uid + self.command = fetch_spec + return "OK", [(b"1 (UID 1 RFC822.SIZE 11)", b"12345678901")] + + client = Client() + with patch.dict("os.environ", {"GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES": "10"}), self.assertRaisesRegex( + ImapAppendError, + "deployment limit", + ): + _fetch_message_by_uid(client, "1") + self.assertIn("BODY.PEEK[]<0.11>", client.command) + def test_paginates_sequence_numbers_newest_first(self): self.assertEqual( _paged_descending_sequences(120, offset=0, limit=5), diff --git a/tests/test_smtp_helpers.py b/tests/test_smtp_helpers.py index 2775a28..ca4c20a 100644 --- a/tests/test_smtp_helpers.py +++ b/tests/test_smtp_helpers.py @@ -1,16 +1,44 @@ from __future__ import annotations import unittest +from unittest.mock import patch +from govoplan_core.security.outbound_http import OutboundHttpBlocked from govoplan_mail.backend.config import SmtpConfig from govoplan_mail.backend.sending.smtp import ( SmtpConfigurationError, + _open_smtp, _prepare_smtp_send, _smtp_send_result, ) class SmtpSendHelperTests(unittest.TestCase): + def test_real_smtp_connections_honor_deployment_egress_policy(self): + config = SmtpConfig(host="smtp.internal", port=587, security="starttls") + with patch( + "govoplan_mail.backend.sending.smtp.validate_outbound_host", + side_effect=OutboundHttpBlocked("private network blocked"), + ), self.assertRaisesRegex(SmtpConfigurationError, "private network blocked"): + _open_smtp(config) + + def test_smtp_revalidates_and_pins_at_connection_time(self): + config = SmtpConfig(host="smtp.example.test", port=587, security="starttls") + public = [(2, 1, 6, "", ("93.184.216.34", 587))] + private = [(2, 1, 6, "", ("127.0.0.1", 587))] + with patch.dict( + "os.environ", + {"APP_ENV": "production", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "false"}, + ), patch( + "govoplan_core.security.outbound_http.socket.getaddrinfo", + side_effect=(public, private), + ), patch("govoplan_core.security.outbound_http.socket.socket") as socket_factory, self.assertRaisesRegex( + OutboundHttpBlocked, + "non-public network", + ): + _open_smtp(config) + socket_factory.assert_not_called() + def test_prepare_smtp_send_validates_envelope(self): config = SmtpConfig(host="smtp.example.org", port=587, security="starttls") with self.assertRaisesRegex(SmtpConfigurationError, "envelope sender"): @@ -45,4 +73,3 @@ class SmtpSendHelperTests(unittest.TestCase): if __name__ == "__main__": unittest.main() -