From 218fef11f1b9879bdd708c136f1967fde8289735 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 22 Aug 2026 04:52:25 +0200 Subject: [PATCH] feat(mail): add governed POP3 legacy import --- README.md | 22 +- docs/MAIL_HANDBOOK.md | 92 ++- docs/MAIL_PROTOCOL_ROADMAP.md | 38 +- package-lock.json | 4 +- package.json | 2 +- pyproject.toml | 2 +- src/govoplan_mail/backend/config.py | 41 ++ src/govoplan_mail/backend/db/models.py | 69 ++ src/govoplan_mail/backend/dsar_provider.py | 85 +++ src/govoplan_mail/backend/manifest.py | 175 ++++- .../a4c5d6e7f809_mail_pop3_imports.py | 91 +++ src/govoplan_mail/backend/pop3_imports.py | 246 +++++++ src/govoplan_mail/backend/provider_state.py | 135 ++++ src/govoplan_mail/backend/router.py | 387 ++++++++++- src/govoplan_mail/backend/schemas.py | 82 ++- src/govoplan_mail/backend/sending/pop3.py | 492 ++++++++++++++ src/govoplan_mail/backend/server_hierarchy.py | 46 +- tests/test_dsar_provider.py | 45 ++ tests/test_manifest.py | 34 + tests/test_pop3_imports.py | 440 +++++++++++++ tests/test_provider_state.py | 80 ++- webui/package-lock.json | 4 +- webui/package.json | 2 +- .../test-interface-pattern-language.mjs | 15 +- webui/src/api/mail.ts | 130 +++- .../features/mail/MailLegacyImportPage.tsx | 610 ++++++++++++++++++ webui/src/module.ts | 10 +- 27 files changed, 3291 insertions(+), 88 deletions(-) create mode 100644 src/govoplan_mail/backend/migrations/versions/a4c5d6e7f809_mail_pop3_imports.py create mode 100644 src/govoplan_mail/backend/pop3_imports.py create mode 100644 src/govoplan_mail/backend/sending/pop3.py create mode 100644 tests/test_pop3_imports.py create mode 100644 webui/src/features/mail/MailLegacyImportPage.tsx diff --git a/README.md b/README.md index 92d4712..4835db0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ **Repository type:** module (domain). -GovOPlaN Mail is the mail transport module. It owns reusable SMTP/IMAP profile management, mail profile policy enforcement, mock mail infrastructure, and the mail WebUI package. +GovOPlaN Mail is the mail transport module. It owns reusable SMTP/IMAP profile management, an explicitly enabled legacy POP3 import path, mail profile policy enforcement, mock mail infrastructure, and the mail WebUI package. ## Ownership @@ -12,17 +12,18 @@ This repository owns: - backend module manifest `mail` - mail permissions such as `mail:profile:read`, `mail:profile:write_own`, `mail:profile:write`, `mail:profile:use`, `mail:profile:test`, and `mail:mailbox:read` -- SMTP/IMAP profile models, policy checks, encrypted credential storage, and profile resolution +- SMTP/IMAP profile models, dedicated legacy POP3 sources, policy checks, encrypted credential storage, and profile resolution - SMTP send and IMAP append adapters, including mock transports for development - development mock mailbox endpoints used by test-send flows -- WebUI package `@govoplan/mail-webui` with profile management, policy management, and read-only mailbox components +- WebUI package `@govoplan/mail-webui` with profile management, policy management, read-only mailbox components, and governed POP3 import Core owns auth, tenants, RBAC evaluation, database/session primitives, secret helpers, CSRF/API helpers, and shell layout. Mail publishes `privacy.dsar.mail` for Core's governed data-subject-request workflow. It isolates matching mailbox header parties and returns bounded index, -personal-profile, delivery, reconciliation, and bounce metadata. SMTP/IMAP -configuration and credentials, encrypted messages and envelopes, folder/UID +personal-profile, delivery, reconciliation, bounce, and imported-message metadata. SMTP/IMAP/POP3 +configuration and credentials, encrypted messages and envelopes, source UIDL +and folder/UID locators, worker and idempotency state, diagnostics, and opaque evidence are excluded. Delivery and bounce outcomes remain retained evidence; mailbox and profile changes require coordinated Mail and external-provider review, so the @@ -143,10 +144,13 @@ Development mailbox routes are registered by the mail module only when the core runtime is in `dev` mode and `dev_mailbox_api_enabled` is enabled. Core does not contribute these routes directly. -POP3 and JMAP are deferred. The protocol decision is documented in -[docs/MAIL_PROTOCOL_ROADMAP.md](docs/MAIL_PROTOCOL_ROADMAP.md): stabilize -SMTP/IMAP first, prefer JMAP for modern mailbox sync/search later, and add POP3 -only for explicit legacy-download requirements. The same roadmap records the +JMAP remains deferred. The explicitly enabled POP3 slice is limited to bounded, +encrypted, duplicate-safe legacy import; preview and ordinary import are +non-destructive, while source deletion requires separate endpoint policy, +permission, confirmation, and audit evidence. The protocol decision is +documented in [docs/MAIL_PROTOCOL_ROADMAP.md](docs/MAIL_PROTOCOL_ROADMAP.md): +prefer JMAP for future modern mailbox sync/search and keep POP3 out of normal +mailbox browsing. The same roadmap records the approved S/MIME-first, OpenPGP-additional message-protection profile and its no-silent-downgrade requirement. diff --git a/docs/MAIL_HANDBOOK.md b/docs/MAIL_HANDBOOK.md index 92f1ab8..beb595b 100644 --- a/docs/MAIL_HANDBOOK.md +++ b/docs/MAIL_HANDBOOK.md @@ -23,12 +23,13 @@ See also [Mail protocol roadmap](MAIL_PROTOCOL_ROADMAP.md) and the Campaign Mail owns: -- reusable SMTP/IMAP profile definitions and scope; -- encrypted SMTP/IMAP credentials and safe credential replacement; +- reusable SMTP/IMAP profile definitions, dedicated legacy POP3 sources, and scope; +- encrypted SMTP/IMAP/POP3 credentials and safe credential replacement; - effective profile policy and visibility/authorization decisions; - connection tests and protocol adapters; - SMTP send and IMAP append operations exposed to consumers; -- read-only mailbox folder/message access and its bounded indexes; and +- read-only mailbox folder/message access, bounded indexes, and encrypted + pending-review records imported from legacy POP3 sources; and - the transport sanitization boundary and throttling behavior. A general Mail-owned provider-attempt/diagnostic ledger remains planned. @@ -65,6 +66,10 @@ profile test and Ops health surfaces. If the receipt says SMTP is unavailable, is invalid, or is not mounted, import is blocked with an operator-facing resolution instead of creating a partial profile. +Configuration packages remain SMTP-focused. A POP3 legacy source is a deliberate +operational migration action and is not silently exported, cloned, or enabled by +an SMTP profile package. + ## Interface patterns and unavailable actions Mail uses the platform's shared explorer, connection tree, adaptive form, @@ -89,9 +94,10 @@ deactivating a profile may scrub Mail-owned credentials as described below. ### Profile A profile is a reusable, named delivery identity with optional SMTP and IMAP -configuration. It has a stable id, lifecycle state, scope, owner context, and -non-secret connection metadata. Passwords are write-only encrypted values and -are never returned through list/read/capability responses. +configuration. It can additionally own a dedicated POP3 endpoint for an +explicit legacy-import workflow. It has a stable id, lifecycle state, scope, +owner context, and non-secret connection metadata. Passwords are write-only +encrypted values and are never returned through list/read/capability responses. An IMAP server may map the standard Inbox, Sent, Drafts, Trash, Archive, and Junk roles to exact provider folder names. These mappings belong to the reusable @@ -128,6 +134,11 @@ authorization. Mail re-evaluates profile activity, visibility, policy, and revision immediately before it resolves credentials and performs the effect. +POP3 imports pin the endpoint/credential transport revision at preview time. A +changed revision or a missing provider UIDL stops import and requires a fresh +preview. POP3 does not participate in the ordinary mailbox folder/message +projection. + ### Provider outcomes SMTP acceptance, partial or complete recipient refusal, temporary/permanent @@ -202,6 +213,46 @@ not mutate read/unread, delete, move, or reply state. Message responses are bounded by the deployment response policy; ordinary UI should avoid loading a whole large mailbox or attachment merely to show a list. +### Import a legacy POP3 mailbox + +POP3 is available only for bounded migration from a legacy server that cannot +provide IMAP or JMAP. It is disabled by default and is not a replacement for +the read-only mailbox UI. + +1. An actor with profile-write, secret-management, and `mail:pop3:manage` + authority opens **Legacy POP3 import**, selects a Mail profile, and creates a + dedicated source. The UI stages the endpoint disabled, stores its encrypted + username/password credential, and enables it only after both operations + succeed. +2. The administrator explicitly enables legacy import, sets TLS mode, timeout, + maximum message and batch sizes, and preview body lines, and tests connection, + authentication, TLS, and provider message count. Plain transport remains + subject to deployment egress/security policy and should not be used across + an untrusted network. +3. An operator with `mail:profile:use` and `mail:pop3:import` refreshes a live + preview of at most 100 messages. Preview sends no `DELE`, changes no flags, + and exposes only bounded headers/body text. A server without stable UIDL + identifiers is rejected. If `TOP` is unavailable, Mail uses `RETR` only + inside the configured size bound and suppresses an oversized preview. +4. The operator selects messages. Mail downloads within the size gate and + creates encrypted local `pending_review` records. Tenant, profile, endpoint, + and UIDL form the duplicate boundary. Raw content never appears in list, + provider-state, audit, or DSAR output. +5. Source messages remain untouched by default. Delete-after-import requires + the endpoint's separate `allow_delete_after_import` policy, the operator's + `mail:pop3:delete` permission, an explicit per-batch choice, and destructive + confirmation. Mail commits the local import plus `mail.pop3.imported` audit + evidence before sending `DELE`. It separately records + `mail.pop3.source_deletion`; disconnect during `QUIT` is outcome-unknown and + must be reconciled before another destructive attempt. + +The supplied **Mail legacy import operator** role can test, preview, and import +without deleting. The **Mail profile administrator** role also contains source +management and destructive-delete permissions; deployments should remove or +split `mail:pop3:delete` when operators must never delete provider messages. +Imported records follow configured Mail/records retention and require manual +review for a data-subject request or deletion decision. + ## Profile administration ### Roles @@ -214,11 +265,16 @@ The supplied templates are: and manage credentials only for the current account's own user-scoped profiles, subject to the effective Mail policy. - **Mail profile administrator:** additionally create/update profiles and - create/replace encrypted credentials across tenant-owned scopes. + create/replace encrypted credentials across tenant-owned scopes, configure + legacy POP3 imports, and—unless the template is narrowed—request source + deletion after import. +- **Mail legacy import operator:** test approved POP3 sources and preview/import + messages without permission to delete them at the provider. The specific permissions are `mail:profile:read`, `mail:profile:use`, `mail:profile:test`, `mail:mailbox:read`, `mail:profile:write_own`, -`mail:secret:manage_own`, `mail:profile:write`, and `mail:secret:manage`. +`mail:secret:manage_own`, `mail:profile:write`, `mail:secret:manage`, +`mail:pop3:manage`, `mail:pop3:import`, and `mail:pop3:delete`. The `_own` permissions are enforced against the authenticated membership id and never authorize a tenant, group, campaign, system, or another user's profile. They also do not authorize profile-policy changes. System-scoped definitions @@ -305,10 +361,10 @@ tenant posture; it does not expose credential material. Private-network connector access is controlled deployment-wide by `GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS`. Whether private or public targets -are allowed, SMTP and IMAP resolve, validate, and connect to the exact approved -address records at connection time while retaining the original hostname for -TLS SNI and certificate verification. A DNS change cannot redirect the socket -after validation. +are allowed, SMTP, IMAP, and legacy POP3 resolve, validate, and connect to the +exact approved address records at connection time while retaining the original +hostname for TLS SNI and certificate verification. A DNS change cannot redirect +the socket after validation. Transports that cannot pin every connection peer or revalidate protocol-managed redirects/referrals must fail before client construction. Do not weaken this @@ -383,10 +439,11 @@ state transition or outbound synchronization effect. ### Backup, restore, and retirement -Backups contain encrypted credentials and therefore need the same protection as -the live database and key material. Restoring a Mail database without the -matching encryption key makes credentials unusable; restoring it with keys can -reactivate sensitive historical state and must be controlled. +Backups contain encrypted credentials and encrypted raw POP3 import records and +therefore need the same protection as the live database and key material. +Restoring a Mail database without the matching encryption key makes those +records unusable; restoring it with keys can reactivate sensitive historical +state and must be controlled. Destructive module retirement first applies the same immediate credential scrub/audit rule to every remaining profile, then drops Mail-owned tables after @@ -495,7 +552,8 @@ Before claiming a Mail composition is production-ready: Campaign business-action contract built on Mail transport operations. - JMAP mailbox synchronization/search; it is preferred only after the IMAP MVP is stable. -- POP3 except for a future explicit legacy download/import requirement. +- Expanding POP3 beyond the implemented explicit legacy download/import + workflow; it has no folder, flag, search, or synchronization contract. - A full mail client with compose/reply/move/delete/read-state mutation. - Quick Access may launch the operating environment's configured composer via `mailto:`. That explicit handoff is not a GovOPlaN Mail delivery: it selects diff --git a/docs/MAIL_PROTOCOL_ROADMAP.md b/docs/MAIL_PROTOCOL_ROADMAP.md index 6fe7dad..ce88836 100644 --- a/docs/MAIL_PROTOCOL_ROADMAP.md +++ b/docs/MAIL_PROTOCOL_ROADMAP.md @@ -1,12 +1,15 @@ # Mail Protocol Roadmap -GovOPlaN Mail currently focuses on SMTP sending and IMAP mailbox access. POP3 -and JMAP are deferred until the IMAP mailbox MVP is stable. +GovOPlaN Mail focuses on SMTP sending and IMAP mailbox access. It also provides +an explicitly enabled, bounded POP3 legacy-import path. JMAP remains deferred +until the IMAP mailbox MVP and protocol-neutral mailbox contract are stable. ## Current Baseline - SMTP is the send protocol. - IMAP is the read/append protocol. +- POP3 is an optional legacy migration source, never a general mailbox + protocol or default profile endpoint. - Mail profile policy, encrypted credentials, mailbox folder parsing, test buttons, and read-only mailbox UI are built around SMTP and IMAP. @@ -56,17 +59,30 @@ JMAP should be added only after: ## POP3 -POP3 should remain legacy-only. +POP3 remains legacy-only. The bounded import slice is available when a concrete +deployment must retire a mailbox that cannot offer IMAP or JMAP. -Add it only when a concrete deployment requires mailbox download from a server -that cannot offer IMAP or JMAP. POP3 is a poor fit for the normal GovOPlaN -mailbox UX because it has limited folder, sync, and server-side state semantics. +It is disabled until an administrator creates a dedicated POP3 endpoint and +sets `legacy_import_enabled`. The endpoint has its own encrypted credential, +connection/TLS/authentication diagnostics, maximum message and batch sizes, preview body +limit, and a separate `allow_delete_after_import` policy. Stable UIDL support is +mandatory; Mail refuses import when a provider cannot supply it. -If implemented, POP3 should be scoped to explicit download/import workflows, not -general mailbox browsing. +Preview and ordinary import are non-destructive. Selected messages become +encrypted `pending_review` records with a content digest, pinned transport +revision, source UIDL, and audit evidence. Repeating a UIDL reports a duplicate. +Provider deletion requires both endpoint policy and `mail:pop3:delete`, is +chosen separately per batch, and runs only after the local import and its audit +event commit. A disconnect while POP3 `QUIT` commits deletions becomes +`outcome_unknown` and is never retried blindly. + +POP3 does not supply folder, flag, thread, search, or synchronization semantics. +It is therefore excluded from the normal mailbox UI and from the recommended +ongoing Mail profile. Configuration-package export/import remains SMTP-focused; +legacy source rollout is an explicit operational action. ## Decision -Do not add POP3 or JMAP now. Stabilize SMTP/IMAP first, design protocol-neutral -mailbox DTOs, then prefer JMAP for modern servers and reserve POP3 for explicit -legacy download requirements. +Keep the implemented POP3 surface limited to governed legacy import. Do not +expand it into mailbox browsing. Design protocol-neutral mailbox DTOs and +prefer JMAP for future modern synchronization/search support. diff --git a/package-lock.json b/package-lock.json index 466496f..bd0b32b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@govoplan/mail-webui", - "version": "0.1.18", + "version": "0.1.20", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govoplan/mail-webui", - "version": "0.1.18", + "version": "0.1.20", "peerDependencies": { "@govoplan/core-webui": "^0.1.18", "lucide-react": "^1.23.0", diff --git a/package.json b/package.json index 8e37639..e2a19f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/mail-webui", - "version": "0.1.18", + "version": "0.1.20", "private": true, "type": "module", "main": "webui/src/index.ts", diff --git a/pyproject.toml b/pyproject.toml index ead59bd..490d0c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-mail" -version = "0.1.19" +version = "0.1.20" description = "GovOPlaN mail module with backend and WebUI integration." readme = "README.md" requires-python = ">=3.12" diff --git a/src/govoplan_mail/backend/config.py b/src/govoplan_mail/backend/config.py index d078025..92c352d 100644 --- a/src/govoplan_mail/backend/config.py +++ b/src/govoplan_mail/backend/config.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pydantic import Field, model_validator + from govoplan_core.mail.config import ( ImapConfig, ImapFolderMappings, @@ -12,10 +14,49 @@ from govoplan_core.mail.config import ( normalize_split_transport_credentials, ) + +class Pop3ServerConfig(StrictModel): + """Server-only settings for an explicitly enabled legacy POP3 source.""" + + host: str | None = None + port: int | None = Field(default=None, ge=1, le=65535) + security: TransportSecurity = TransportSecurity.TLS + timeout_seconds: int = Field(default=30, ge=1, le=300) + max_message_bytes: int = Field(default=25 * 1024 * 1024, ge=1_024, le=50 * 1024 * 1024) + max_batch_bytes: int = Field(default=100 * 1024 * 1024, ge=1_048_576, le=500 * 1024 * 1024) + preview_body_lines: int = Field(default=20, ge=0, le=100) + legacy_import_enabled: bool = False + allow_delete_after_import: bool = False + + @model_validator(mode="after") + def apply_default_port(self) -> "Pop3ServerConfig": + if self.port is None: + self.port = 995 if self.security == TransportSecurity.TLS else 110 + if self.legacy_import_enabled and not str(self.host or "").strip(): + raise ValueError( + "POP3 host is required when legacy import is enabled" + ) + if self.max_batch_bytes < self.max_message_bytes: + raise ValueError( + "POP3 batch size limit cannot be lower than the per-message limit" + ) + if self.allow_delete_after_import and not self.legacy_import_enabled: + raise ValueError( + "POP3 delete-after-import cannot be enabled while legacy import is disabled" + ) + return self + + +class Pop3Config(Pop3ServerConfig): + username: str | None = None + password: str | None = None + __all__ = [ "ImapConfig", "ImapFolderMappings", "ImapServerConfig", + "Pop3Config", + "Pop3ServerConfig", "SmtpConfig", "SmtpServerConfig", "StrictModel", diff --git a/src/govoplan_mail/backend/db/models.py b/src/govoplan_mail/backend/db/models.py index 99f9d41..813a30b 100644 --- a/src/govoplan_mail/backend/db/models.py +++ b/src/govoplan_mail/backend/db/models.py @@ -387,3 +387,72 @@ class MailBounceObservation(Base, TimestampMixin): ) matched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + + +class MailPop3Import(Base, TimestampMixin): + """Governed local review record created from a legacy POP3 mailbox.""" + + __tablename__ = "mail_pop3_imports" + __table_args__ = ( + UniqueConstraint( + "tenant_id", + "profile_id", + "pop3_server_id", + "provider_uidl", + name="uq_mail_pop3_imports_source_uidl", + ), + Index( + "ix_mail_pop3_imports_review", + "tenant_id", + "status", + "imported_at", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid) + tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True) + profile_id: Mapped[str] = mapped_column( + ForeignKey("mail_server_profiles.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + pop3_server_id: Mapped[str] = mapped_column( + ForeignKey("mail_server_endpoints.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + pop3_credential_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + transport_revision: Mapped[str] = mapped_column(String(120), nullable=False) + provider_uidl: Mapped[str] = mapped_column(String(500), nullable=False) + provider_message_number: Mapped[int | None] = mapped_column(Integer, nullable=True) + fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + raw_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + raw_message_encrypted: Mapped[str] = mapped_column(Text, nullable=False) + message_id: Mapped[str | None] = mapped_column(String(998), nullable=True, index=True) + subject: Mapped[str | None] = mapped_column(Text, nullable=True) + from_header: Mapped[str | None] = mapped_column(Text, nullable=True) + to_header: Mapped[str | None] = mapped_column(Text, nullable=True) + date: Mapped[str | None] = mapped_column(String(255), nullable=True) + body_preview: Mapped[str | None] = mapped_column(Text, nullable=True) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + status: Mapped[str] = mapped_column( + String(40), default="pending_review", nullable=False, index=True + ) + imported_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, index=True + ) + imported_by_user_id: Mapped[str | None] = mapped_column( + ForeignKey("access_users.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + deletion_requested: Mapped[bool] = mapped_column( + Boolean, default=False, nullable=False + ) + deletion_status: Mapped[str] = mapped_column( + String(40), default="not_requested", nullable=False, index=True + ) + deletion_attempted_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + deletion_error: Mapped[str | None] = mapped_column(String(500), nullable=True) diff --git a/src/govoplan_mail/backend/dsar_provider.py b/src/govoplan_mail/backend/dsar_provider.py index b4dc22f..249f618 100644 --- a/src/govoplan_mail/backend/dsar_provider.py +++ b/src/govoplan_mail/backend/dsar_provider.py @@ -20,6 +20,7 @@ from govoplan_mail.backend.db.models import ( MailDeliveryCommand, MailDeliveryReconciliation, MailMailboxMessageIndex, + MailPop3Import, MailServerProfile, ) @@ -116,6 +117,38 @@ class MailDsarProvider: ) ) + pop3_imports = _matching_pop3_imports( + db, + tenant_id=tenant_id, + email=email, + import_id=references.get("pop3_import"), + ) + for imported in pop3_imports: + matching_headers = _matching_pop3_headers(imported, email) + append( + _record( + "mail_pop3_import", + imported.id, + "mail_imported_message", + imported.subject or "Imported legacy message", + { + "match_fields": list(matching_headers), + "subject": _bounded_text(imported.subject), + "matching_headers": matching_headers, + "date": imported.date, + "message_id": _bounded_text(imported.message_id), + "body_preview": _bounded_text(imported.body_preview), + "size_bytes": imported.size_bytes, + "status": imported.status, + "imported_at": _iso(imported.imported_at), + "deletion_requested": imported.deletion_requested, + "deletion_status": imported.deletion_status, + }, + observed_at=imported.updated_at, + source_path="/mail/legacy-import", + ) + ) + bounces = _matching_bounces( db, tenant_id=tenant_id, @@ -404,6 +437,57 @@ def _matching_bounces( ) +def _matching_pop3_imports( + session: Session, + *, + tenant_id: str, + email: str | None, + import_id: str | None, +) -> list[MailPop3Import]: + conditions = [] + if import_id: + conditions.append(MailPop3Import.id == import_id) + if email: + pattern = f"%{_escape_like(email)}%" + conditions.extend( + func.lower(field).like(pattern, escape="\\") + for field in ( + MailPop3Import.from_header, + MailPop3Import.to_header, + ) + ) + if not conditions: + return [] + candidates = _bounded_rows( + session.query(MailPop3Import) + .filter(MailPop3Import.tenant_id == tenant_id, or_(*conditions)) + .order_by(MailPop3Import.id) + ) + return [ + row + for row in candidates + if row.id == import_id or _matching_pop3_headers(row, email) + ] + + +def _matching_pop3_headers( + row: MailPop3Import, + email: str | None, +) -> dict[str, list[dict[str, str | None]]]: + if email is None: + return {} + result = {} + for role, value in (("from", row.from_header), ("to", row.to_header)): + matches = [ + {"email": address.casefold(), "name": name or None} + for name, address in getaddresses([value or ""]) + if address.casefold() == email + ] + if matches: + result[role] = matches[:64] + return result + + def _matching_commands( session: Session, *, @@ -458,6 +542,7 @@ def _mail_references(subject: DsarSubjectRef) -> dict[str, str]: "mail.message_index": "message_index", "mail.delivery_command": "command", "mail.bounce_observation": "bounce", + "mail.pop3_import": "pop3_import", } return { target: value diff --git a/src/govoplan_mail/backend/manifest.py b/src/govoplan_mail/backend/manifest.py index b8847c3..a8689c8 100644 --- a/src/govoplan_mail/backend/manifest.py +++ b/src/govoplan_mail/backend/manifest.py @@ -50,8 +50,10 @@ from govoplan_mail.backend.documentation import ( ) from govoplan_mail.backend.provider_state import ( IMAP_PROVIDER_ID, + POP3_PROVIDER_ID, SMTP_PROVIDER_ID, imap_provider_states, + pop3_provider_states, smtp_provider_states, ) from govoplan_mail.backend.db import models as mail_models # noqa: F401 - populate Mail ORM metadata @@ -66,6 +68,7 @@ _mail_table_retirement_provider = drop_table_retirement_provider( mail_models.MailProfilePolicy, mail_models.MailMailboxFolderIndex, mail_models.MailMailboxMessageIndex, + mail_models.MailPop3Import, mail_models.MailDeliveryReconciliation, mail_models.MailDeliveryAttempt, mail_models.MailDeliveryCommand, @@ -94,7 +97,7 @@ def _mail_retirement_provider(session: object | None, module_id: str): plan, destroy_data_warnings=( *plan.destroy_data_warnings, - "Mail-owned encrypted SMTP/IMAP passwords are scrubbed with non-secret audit records immediately before tables are dropped; any scrub or audit failure blocks retirement.", + "Mail-owned encrypted SMTP/IMAP/POP3 passwords are scrubbed with non-secret audit records immediately before tables are dropped; encrypted POP3 import records are then removed with the Mail tables. Any scrub or audit failure blocks retirement.", ), destroy_data_executor=executor, ) @@ -129,9 +132,9 @@ def _permission(scope: str, label: str, description: str) -> PermissionDefinitio PERMISSIONS = ( - _permission("mail:profile:read", "View mail profiles", "Inspect reusable SMTP/IMAP profile metadata."), + _permission("mail:profile:read", "View mail profiles", "Inspect reusable SMTP/IMAP and governed legacy POP3 profile metadata."), _permission("mail:profile:use", "Use mail profiles", "Select an approved mail profile for delivery."), - _permission("mail:profile:test", "Test mail profiles", "Run SMTP/IMAP connection tests."), + _permission("mail:profile:test", "Test mail profiles", "Run SMTP/IMAP connection tests and explicitly enabled POP3 diagnostics."), _permission("mail:mailbox:read", "Read mailboxes", "List IMAP folders and inspect messages without mutating mailbox state."), _permission("mail:profile:write", "Manage mail profiles", "Create and edit reusable mail profiles."), _permission( @@ -139,7 +142,22 @@ PERMISSIONS = ( "Manage own mail profiles", "Create, edit, and deactivate only the current account's user-scoped mail profiles within effective policy.", ), - _permission("mail:secret:manage", "Manage mail secrets", "Create or replace stored SMTP/IMAP credentials."), + _permission("mail:secret:manage", "Manage mail secrets", "Create or replace stored SMTP/IMAP/POP3 credentials."), + _permission( + "mail:pop3:manage", + "Manage legacy POP3 imports", + "Configure explicitly enabled legacy POP3 sources and their non-destructive or delete-after-import policy.", + ), + _permission( + "mail:pop3:import", + "Import legacy POP3 messages", + "Preview and import bounded messages from an explicitly enabled legacy POP3 source.", + ), + _permission( + "mail:pop3:delete", + "Delete imported POP3 messages at source", + "Request source deletion after a governed local import when the server policy also allows it.", + ), _permission( "mail:delivery:diagnostic", "Inspect mail delivery diagnostics", @@ -183,6 +201,20 @@ ROLE_TEMPLATES = ( "mail:delivery:reconcile", "mail:bounce:read", "mail:bounce:manage", + "mail:pop3:manage", + "mail:pop3:import", + "mail:pop3:delete", + ), + ), + RoleTemplate( + slug="mail_legacy_import_operator", + name="Mail legacy import operator", + description="Preview and import from an approved POP3 legacy source without deleting provider messages.", + permissions=( + "mail:profile:read", + "mail:profile:use", + "mail:profile:test", + "mail:pop3:import", ), ), RoleTemplate( @@ -317,10 +349,58 @@ IMAP_PROVIDER = ExternalProviderDeclaration( ) +POP3_PROVIDER = ExternalProviderDeclaration( + id=POP3_PROVIDER_ID, + module_id="mail", + label="Legacy POP3 governed import", + maturity="publish", + operations=("discover", "read", "preview", "write", "delete"), + objects=( + ProviderObjectDeclaration( + object_type="legacy_mailbox_message", + field_groups=("provider_identity", "safe_headers", "body_preview", "size"), + authority_modes=("external_authoritative", "governance_overlay"), + default_authority_mode="external_authoritative", + ), + ProviderObjectDeclaration( + object_type="mail_pop3_import", + field_groups=("source_identity", "content_digest", "encrypted_content", "review_state", "deletion_evidence"), + authority_modes=("governance_overlay",), + default_authority_mode="governance_overlay", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="Each import pins the explicit POP3 endpoint and credential transport revision and identifies source messages by UIDL.", + concurrency="The expected transport revision is checked before credentials are decrypted; missing or changed UIDLs require a fresh preview.", + freshness="POP3 previews are live bounded observations; imported records retain their source UIDL and import time.", + health="Disabled, enabled, imported, and failed or unknown source-deletion states are projected without exposing secrets.", + max_read_items=100, + idempotency="Tenant, profile, server, and provider UIDL uniquely identify one governed local import.", + retry="Preview and pre-effect download failures are safe to retry; source deletion is never retried blindly after QUIT begins.", + timeout_seconds=60, + conflicts="A repeated UIDL is reported as an existing import and cannot create duplicate local content.", + outcome_unknown="A connection failure while committing POP3 DELE operations is retained as outcome_unknown for operator reconciliation.", + outcome_unknown_supported=True, + evidence="Encrypted raw message content, bounded safe headers, content digest, UIDL, pinned revision, import audit, and separate deletion outcome are retained.", + audit_event_types=("mail.pop3.imported", "mail.pop3.source_deletion"), + correction="Operators review the local import; later source or classification corrections do not rewrite import evidence.", + rollback="A committed local import is preserved even if optional provider deletion fails or is uncertain.", + compensation="Operators reconcile the source mailbox before any further destructive action when deletion is uncertain.", + reconciliation="Compare the retained UIDL and digest with the provider mailbox and record an operational decision outside blind automation.", + outage="No provider effect occurs while unavailable; already imported encrypted records remain available for governed review.", + classifications=("confidential", "personal", "special_category"), + purposes=("legacy mailbox migration", "governed message intake"), + retention="Imported content and source-deletion evidence follow configured Mail and records-retention policy.", + secret_handling="POP3 credentials and imported raw messages are encrypted; neither is returned by preview, import-list, provider-state, or DSAR APIs.", + ), + documentation_topic_ids=("mail.workflow.legacy-pop3-import",), +) + + manifest = ModuleManifest( id="mail", name="Mail", - version="0.1.19", + version="0.1.20", required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR), optional_dependencies=("campaigns", "addresses", "calendar", "postbox", "search"), provides_interfaces=( @@ -385,7 +465,10 @@ manifest = ModuleManifest( factory=create_mail_search_source, ), ), - nav_items=(NavItem(path="/mail", label="Mail", icon="mail", required_any=("mail:mailbox:read", "mail:bounce:read", "mail:bounce:manage"), order=50),), + nav_items=( + NavItem(path="/mail", label="Mail", icon="mail", required_any=("mail:mailbox:read", "mail:bounce:read", "mail:bounce:manage"), order=50), + NavItem(path="/mail/legacy-import", label="Legacy POP3 import", icon="mail", required_any=("mail:pop3:import", "mail:pop3:manage"), order=52), + ), frontend=FrontendModule( module_id="mail", package_name="@govoplan/mail-webui", @@ -403,8 +486,18 @@ manifest = ModuleManifest( order=51, surface_id="mail.bounce-processing", ), + FrontendRoute( + path="/mail/legacy-import", + component="MailLegacyImportPage", + required_any=("mail:pop3:import", "mail:pop3:manage"), + order=52, + surface_id="mail.legacy-pop3-import", + ), + ), + nav_items=( + NavItem(path="/mail", label="Mail", icon="mail", required_any=("mail:mailbox:read", "mail:bounce:read", "mail:bounce:manage"), order=50), + NavItem(path="/mail/legacy-import", label="Legacy POP3 import", icon="mail", required_any=("mail:pop3:import", "mail:pop3:manage"), order=52), ), - nav_items=(NavItem(path="/mail", label="Mail", icon="mail", required_any=("mail:mailbox:read", "mail:bounce:read", "mail:bounce:manage"), order=50),), view_surfaces=( ViewSurface(id="mail.admin.system-servers", module_id="mail", kind="section", label="System mail servers", order=70), ViewSurface(id="mail.admin.tenant-servers", module_id="mail", kind="section", label="Tenant mail servers", order=60), @@ -458,6 +551,7 @@ manifest = ModuleManifest( mail_models.MailProfilePolicy, mail_models.MailMailboxFolderIndex, mail_models.MailMailboxMessageIndex, + mail_models.MailPop3Import, mail_models.MailDeliveryReconciliation, mail_models.MailDeliveryAttempt, mail_models.MailDeliveryCommand, @@ -490,20 +584,62 @@ manifest = ModuleManifest( capability_documentation={ MAIL_DSAR_CAPABILITY: CapabilityDocumentation( label="Mail data-subject request provider", - summary="Finds isolated mailbox-index, personal-profile, delivery, reconciliation, and bounce metadata without exposing transport secrets.", + summary="Finds isolated mailbox-index, imported-message, personal-profile, delivery, reconciliation, and bounce metadata without exposing transport secrets or encrypted raw content.", contract_version="0.1.0", documentation_types=("admin",), audience=("privacy_officer", "mail_admin", "records_manager"), ), }, documentation=( + DocumentationTopic( + id="mail.workflow.legacy-pop3-import", + title="Import messages from a legacy POP3 mailbox", + summary="Preview and import bounded POP3 messages into encrypted local review records, without deleting source messages by default.", + body=( + "POP3 is a low-priority migration path and is never enabled implicitly. A Mail administrator creates a dedicated POP3 endpoint, enables legacy import, stores an encrypted credential, tests TLS and authentication, and separately decides whether source deletion may ever be requested. An import operator refreshes the live bounded preview, selects messages by stable UIDL, and imports them into encrypted pending-review records. Repeating an import reports duplicates instead of creating more copies. Source messages remain untouched unless the endpoint policy allows delete-after-import, the operator also has mail:pop3:delete, and deletion is explicitly selected for that batch. Mail commits and audits the local import before attempting deletion; failed or uncertain deletion therefore never removes the governed local record and must be reconciled before another destructive attempt. Prefer IMAP or JMAP for ongoing mailbox access." + ), + layer="available", + documentation_types=("user", "admin"), + audience=("mail_admin", "mail_import_operator", "records_manager", "security_reviewer"), + order=34, + conditions=( + DocumentationCondition( + required_modules=("mail",), + any_scopes=("mail:pop3:import", "mail:pop3:manage"), + ), + ), + links=( + DocumentationLink(label="Legacy POP3 import", href="/mail/legacy-import", kind="runtime"), + DocumentationLink(label="Mail handbook", href="govoplan-mail/docs/MAIL_HANDBOOK.md", kind="repository"), + DocumentationLink(label="Protocol roadmap", href="govoplan-mail/docs/MAIL_PROTOCOL_ROADMAP.md", kind="repository"), + ), + related_modules=("audit", "records"), + unlocks=("Governed retirement of legacy POP3 mailboxes without making destructive retrieval the default.",), + metadata={ + "kind": "workflow", + "route": "/mail/legacy-import", + "screen": "Legacy POP3 import", + "steps": [ + "Ask a Mail administrator to configure and explicitly enable a dedicated POP3 legacy source.", + "Test the source and refresh a bounded preview; no message flags or deletion state are changed.", + "Select messages and import them into encrypted pending-review records.", + "Use delete-after-import only where policy and a separate destructive permission allow it, then reconcile failed or unknown outcomes.", + ], + "limitations": [ + "A POP3 server must provide stable UIDL identifiers; otherwise safe duplicate prevention is unavailable and import is refused.", + "POP3 has no folder or flag semantics and is not the recommended protocol for ongoing mailbox access.", + "Source deletion cannot be rolled back and may have an unknown outcome if the connection fails while QUIT commits deletions.", + ], + "verification": "Prove disabled policy, TLS and authentication diagnostics, non-destructive preview/import, UIDL duplicate prevention, encrypted raw retention, separate delete authorization, import-before-delete audit ordering, and failed or outcome-unknown deletion evidence.", + }, + ), DocumentationTopic( id="mail.privacy.data-subject-requests", title="Review Mail data in a data-subject request", summary="Collect tenant-scoped Mail metadata while preserving transport evidence and external-mailbox authority.", body=( - "Mail's DSAR provider searches the effective tenant by normalized header or bounce-recipient email, direct membership references, and namespaced Mail profile, message-index, delivery-command, or bounce references. It isolates only matching From, To, and Cc parties and returns bounded message-index content, safe personal-profile metadata, and delivery, attempt, reconciliation, and bounce outcomes. It excludes SMTP/IMAP configuration, usernames and credentials, encrypted messages and envelopes, refusal detail, mailbox folders and UIDs, endpoint and credential identifiers, idempotency and worker claims, diagnostics, error text, and opaque evidence. " - "Delivery attempts, reconciliation decisions, and bounce observations remain retained as immutable transport and recovery evidence. Mailbox indexes and personal profiles require manual review through Mail and the authoritative external mailbox. The provider performs no direct erasure because deleting a derived index alone would not delete its external source, while changing a profile or delivery command can affect credentials, other users, and preserved evidence." + "Mail's DSAR provider searches the effective tenant by normalized header or bounce-recipient email, direct membership references, and namespaced Mail profile, message-index, POP3-import, delivery-command, or bounce references. It isolates only matching From, To, and Cc parties and returns bounded message-index or imported-message content, safe personal-profile metadata, and delivery, attempt, reconciliation, and bounce outcomes. It excludes SMTP/IMAP/POP3 configuration, usernames and credentials, encrypted messages and envelopes, POP3 source UIDLs and revisions, refusal detail, mailbox folders and UIDs, endpoint and credential identifiers, idempotency and worker claims, diagnostics, error text, and opaque evidence. " + "Delivery attempts, reconciliation decisions, and bounce observations remain retained as immutable transport and recovery evidence. Mailbox indexes, encrypted POP3 imports, and personal profiles require manual review through Mail and the authoritative external mailbox. The provider performs no direct erasure because deleting a derived or imported record alone would not establish the state of its external source, while changing a profile or delivery command can affect credentials, other users, and preserved evidence." ), layer="configured", documentation_types=("admin",), @@ -525,7 +661,7 @@ manifest = ModuleManifest( "route": "/admin?section=tenant-data-subject-requests", "help_contexts": ["admin.privacy.data-subject-requests"], "steps": [ - "Run the Mail provider search and review mailbox-index, profile, delivery, reconciliation, and bounce dispositions.", + "Run the Mail provider search and review mailbox-index, POP3-import, profile, delivery, reconciliation, and bounce dispositions.", "Retain immutable transport evidence with its reason.", "Coordinate approved mailbox content deletion with the authoritative external mailbox and then refresh the derived index.", "Use Mail profile lifecycle controls for approved personal-profile changes; do not edit encrypted payload or evidence rows directly.", @@ -963,8 +1099,8 @@ manifest = ModuleManifest( DocumentationTopic( id="mail.reference.credentials-egress-retirement", title="Protect Mail credentials, network egress, and retirement", - summary="Keep secrets Mail-owned, pin every SMTP/IMAP peer, bound responses, and delete owned credentials immediately with non-secret audit evidence.", - body="Private-network connector access is deployment-wide, but every allowed hostname still resolves to an approved peer that is pinned at socket creation. Unsupported transports fail before connection. Deleting a profile immediately scrubs its owned encrypted SMTP/IMAP passwords and records non-secret audit when secrets existed; a scrub or audit failure rolls the action back. Destructive module retirement applies the same rule before table drop.", + summary="Keep secrets Mail-owned, pin every SMTP/IMAP/POP3 peer, bound responses, and delete owned credentials immediately with non-secret audit evidence.", + body="Private-network connector access is deployment-wide, but every allowed hostname still resolves to an approved peer that is pinned at socket creation. Unsupported transports fail before connection. Deleting a profile immediately scrubs its owned encrypted SMTP/IMAP/POP3 passwords and records non-secret audit when secrets existed; a scrub or audit failure rolls the action back. Encrypted POP3 import content follows Mail/records retention. Destructive module retirement applies the same secret-scrub rule before all Mail tables, including imports, are dropped.", layer="evidence", documentation_types=("admin",), audience=("mail_admin", "platform_operator", "security_reviewer", "release_reviewer"), @@ -1037,7 +1173,7 @@ manifest = ModuleManifest( resolve=documentation_configuration_states, ), ), - external_providers=(SMTP_PROVIDER, IMAP_PROVIDER), + external_providers=(SMTP_PROVIDER, IMAP_PROVIDER, POP3_PROVIDER), external_provider_state_providers=( ExternalProviderStateProviderRegistration( module_id="mail", @@ -1049,6 +1185,11 @@ manifest = ModuleManifest( provider_id=IMAP_PROVIDER_ID, provider=imap_provider_states, ), + ExternalProviderStateProviderRegistration( + module_id="mail", + provider_id=POP3_PROVIDER_ID, + provider=pop3_provider_states, + ), ), architecture=declared_module_architecture( layer="communication_participation", @@ -1056,15 +1197,15 @@ manifest = ModuleManifest( maturity="vertical_slice", documentation_ref="docs/MAIL_HANDBOOK.md", test_ref="tests/test_delivery_outbox.py", - known_limits=("A complete webmail profile and recovery adoption for future provider-side mailbox mutations are not reference-ready.",), + known_limits=("A complete webmail profile and recovery adoption for future provider-side mailbox mutations are not reference-ready; POP3 is supported only as an explicitly enabled, bounded legacy-import path and has no folder or flag semantics.",), supported_authority_modes=( "external_authoritative", "external_mirror", "governance_overlay", ), - owned_concepts=("mail profile", "mail delivery command", "delivery attempt", "mailbox projection"), + owned_concepts=("mail profile", "mail delivery command", "delivery attempt", "mailbox projection", "governed POP3 import"), non_owned_concepts=("campaign", "notification", "recipient address directory", "external mailbox"), - target_tested_providers=(SMTP_PROVIDER_ID, IMAP_PROVIDER_ID), + target_tested_providers=(SMTP_PROVIDER_ID, IMAP_PROVIDER_ID, POP3_PROVIDER_ID), recovery_docs=("docs/MAIL_HANDBOOK.md",), security_docs=("docs/MAIL_HANDBOOK.md",), operations_docs=("docs/MAIL_HANDBOOK.md",), diff --git a/src/govoplan_mail/backend/migrations/versions/a4c5d6e7f809_mail_pop3_imports.py b/src/govoplan_mail/backend/migrations/versions/a4c5d6e7f809_mail_pop3_imports.py new file mode 100644 index 0000000..e545c94 --- /dev/null +++ b/src/govoplan_mail/backend/migrations/versions/a4c5d6e7f809_mail_pop3_imports.py @@ -0,0 +1,91 @@ +"""add governed POP3 legacy imports + +Revision ID: a4c5d6e7f809 +Revises: 93b4c5d6e7f8 +Create Date: 2026-08-22 12:00:00.000000 +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "a4c5d6e7f809" +down_revision = "93b4c5d6e7f8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "mail_pop3_imports", + sa.Column("id", sa.String(length=36), nullable=False), + sa.Column("tenant_id", sa.String(length=36), nullable=False), + sa.Column("profile_id", sa.String(length=36), nullable=False), + sa.Column("pop3_server_id", sa.String(length=36), nullable=False), + sa.Column("pop3_credential_id", sa.String(length=36), nullable=True), + sa.Column("transport_revision", sa.String(length=120), nullable=False), + sa.Column("provider_uidl", sa.String(length=500), nullable=False), + sa.Column("provider_message_number", sa.Integer(), nullable=True), + sa.Column("fingerprint", sa.String(length=64), nullable=False), + sa.Column("raw_sha256", sa.String(length=64), nullable=False), + sa.Column("raw_message_encrypted", sa.Text(), nullable=False), + sa.Column("message_id", sa.String(length=998), nullable=True), + sa.Column("subject", sa.Text(), nullable=True), + sa.Column("from_header", sa.Text(), nullable=True), + sa.Column("to_header", sa.Text(), nullable=True), + sa.Column("date", sa.String(length=255), nullable=True), + sa.Column("body_preview", sa.Text(), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("status", sa.String(length=40), nullable=False), + sa.Column("imported_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("imported_by_user_id", sa.String(length=36), nullable=True), + sa.Column("deletion_requested", sa.Boolean(), nullable=False), + sa.Column("deletion_status", sa.String(length=40), nullable=False), + sa.Column("deletion_attempted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deletion_error", sa.String(length=500), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["profile_id"], ["mail_server_profiles.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["pop3_server_id"], ["mail_server_endpoints.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["imported_by_user_id"], ["access_users.id"], ondelete="SET NULL" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "tenant_id", + "profile_id", + "pop3_server_id", + "provider_uidl", + name="uq_mail_pop3_imports_source_uidl", + ), + ) + for column in ( + "tenant_id", + "profile_id", + "pop3_server_id", + "fingerprint", + "message_id", + "status", + "imported_at", + "imported_by_user_id", + "deletion_status", + ): + op.create_index( + f"ix_mail_pop3_imports_{column}", + "mail_pop3_imports", + [column], + ) + op.create_index( + "ix_mail_pop3_imports_review", + "mail_pop3_imports", + ["tenant_id", "status", "imported_at"], + ) + + +def downgrade() -> None: + op.drop_table("mail_pop3_imports") diff --git a/src/govoplan_mail/backend/pop3_imports.py b/src/govoplan_mail/backend/pop3_imports.py new file mode 100644 index 0000000..eab88e6 --- /dev/null +++ b/src/govoplan_mail/backend/pop3_imports.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import base64 +import hashlib +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Iterable + +from sqlalchemy import or_, select +from sqlalchemy.orm import Session + +from govoplan_core.security.secrets import encrypt_secret +from govoplan_mail.backend.db.models import MailPop3Import, MailServerEndpoint +from govoplan_mail.backend.sending.pop3 import Pop3DownloadedMessage + + +class Pop3ImportError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class Pop3ImportResult: + imported: tuple[MailPop3Import, ...] + duplicates: tuple[MailPop3Import, ...] + + +def create_pop3_imports( + session: Session, + *, + tenant_id: str, + profile_id: str, + pop3_server_id: str, + pop3_credential_id: str | None, + transport_revision: str, + messages: Iterable[Pop3DownloadedMessage], + user_id: str | None, + deletion_requested: bool, +) -> Pop3ImportResult: + downloaded = tuple(messages) + if not downloaded: + raise Pop3ImportError("No POP3 messages were downloaded for import") + uidls = [item.uidl for item in downloaded] + if len(uidls) != len(set(uidls)): + raise Pop3ImportError("The POP3 download contained duplicate UIDL identifiers") + + # Serialize imports per source before checking UIDLs. The database unique + # constraint remains the last line of defense, while this lock lets a + # concurrent request observe the first request's committed rows and report + # them as duplicates instead of surfacing an integrity error. + source = session.scalar( + select(MailServerEndpoint) + .where( + MailServerEndpoint.id == pop3_server_id, + MailServerEndpoint.profile_id == profile_id, + or_( + MailServerEndpoint.tenant_id == tenant_id, + MailServerEndpoint.tenant_id.is_(None), + ), + MailServerEndpoint.protocol == "pop3", + ) + .with_for_update() + ) + if source is None: + raise Pop3ImportError("The selected POP3 source is unavailable") + + existing = { + row.provider_uidl: row + for row in session.scalars( + select(MailPop3Import).where( + MailPop3Import.tenant_id == tenant_id, + MailPop3Import.profile_id == profile_id, + MailPop3Import.pop3_server_id == pop3_server_id, + MailPop3Import.provider_uidl.in_(uidls), + ) + ) + } + imported: list[MailPop3Import] = [] + duplicates: list[MailPop3Import] = [] + now = datetime.now(timezone.utc) + for message in downloaded: + duplicate = existing.get(message.uidl) + if duplicate is not None: + duplicates.append(duplicate) + continue + encrypted = encrypt_secret(base64.b64encode(message.raw).decode("ascii")) + if not encrypted: + raise Pop3ImportError("The downloaded POP3 message could not be encrypted") + summary = message.summary + row = MailPop3Import( + tenant_id=tenant_id, + profile_id=profile_id, + pop3_server_id=pop3_server_id, + pop3_credential_id=pop3_credential_id, + transport_revision=_required_revision(transport_revision), + provider_uidl=message.uidl, + provider_message_number=message.message_number, + fingerprint=_fingerprint( + tenant_id=tenant_id, + profile_id=profile_id, + pop3_server_id=pop3_server_id, + uidl=message.uidl, + raw_sha256=message.raw_sha256, + ), + raw_sha256=message.raw_sha256, + raw_message_encrypted=encrypted, + message_id=summary.message_id, + subject=summary.subject, + from_header=summary.from_header, + to_header=summary.to_header, + date=summary.date, + body_preview=summary.body_preview, + size_bytes=len(message.raw), + status="pending_review", + imported_at=now, + imported_by_user_id=user_id, + deletion_requested=bool(deletion_requested), + deletion_status=("pending" if deletion_requested else "not_requested"), + ) + session.add(row) + imported.append(row) + session.flush() + return Pop3ImportResult(imported=tuple(imported), duplicates=tuple(duplicates)) + + +def list_pop3_imports( + session: Session, + *, + tenant_id: str, + profile_id: str | None = None, + profile_ids: Iterable[str] | None = None, + limit: int = 100, +) -> tuple[MailPop3Import, ...]: + statement = select(MailPop3Import).where( + MailPop3Import.tenant_id == tenant_id + ) + if profile_id: + statement = statement.where(MailPop3Import.profile_id == profile_id) + elif profile_ids is not None: + allowed = tuple(dict.fromkeys(str(value) for value in profile_ids if value)) + if not allowed: + return () + statement = statement.where(MailPop3Import.profile_id.in_(allowed)) + rows = session.scalars( + statement.order_by( + MailPop3Import.imported_at.desc(), + MailPop3Import.id.desc(), + ).limit(max(1, min(int(limit), 500))) + ) + return tuple(rows) + + +def mark_pop3_deletion_result( + session: Session, + *, + tenant_id: str, + import_ids: Iterable[str], + status: str, + error: str | None = None, +) -> tuple[MailPop3Import, ...]: + clean_status = str(status or "").strip().casefold() + if clean_status not in {"succeeded", "failed", "outcome_unknown"}: + raise Pop3ImportError("Unsupported POP3 deletion result") + ids = tuple(dict.fromkeys(str(value).strip() for value in import_ids if str(value).strip())) + if not ids: + return () + rows = tuple( + session.scalars( + select(MailPop3Import) + .where( + MailPop3Import.tenant_id == tenant_id, + MailPop3Import.id.in_(ids), + MailPop3Import.deletion_requested.is_(True), + ) + .with_for_update() + ) + ) + if len(rows) != len(ids): + raise Pop3ImportError("One or more POP3 import records are unavailable") + now = datetime.now(timezone.utc) + safe_error = _bounded_error(error) + for row in rows: + row.deletion_status = clean_status + row.deletion_attempted_at = now + row.deletion_error = safe_error + session.flush() + return rows + + +def pop3_import_payload(row: MailPop3Import) -> dict[str, object]: + return { + "id": row.id, + "profile_id": row.profile_id, + "pop3_server_id": row.pop3_server_id, + "transport_revision": row.transport_revision, + "provider_uidl": row.provider_uidl, + "message_id": row.message_id, + "subject": row.subject, + "from_header": row.from_header, + "to_header": row.to_header, + "date": row.date, + "body_preview": row.body_preview, + "size_bytes": row.size_bytes, + "raw_sha256": row.raw_sha256, + "status": row.status, + "imported_at": row.imported_at, + "deletion_requested": row.deletion_requested, + "deletion_status": row.deletion_status, + "deletion_attempted_at": row.deletion_attempted_at, + "deletion_error": row.deletion_error, + } + + +def _fingerprint( + *, + tenant_id: str, + profile_id: str, + pop3_server_id: str, + uidl: str, + raw_sha256: str, +) -> str: + material = "\x1f".join( + (tenant_id, profile_id, pop3_server_id, uidl, raw_sha256) + ).encode("utf-8") + return hashlib.sha256(material).hexdigest() + + +def _required_revision(value: object) -> str: + clean = str(value or "").strip() + if not clean or len(clean) > 120: + raise Pop3ImportError("A valid POP3 transport revision is required") + return clean + + +def _bounded_error(value: str | None) -> str | None: + clean = " ".join(str(value or "").split()) + return clean[:500] or None + + +__all__ = [ + "Pop3ImportError", + "Pop3ImportResult", + "create_pop3_imports", + "list_pop3_imports", + "mark_pop3_deletion_result", + "pop3_import_payload", +] diff --git a/src/govoplan_mail/backend/provider_state.py b/src/govoplan_mail/backend/provider_state.py index ff444ab..245970c 100644 --- a/src/govoplan_mail/backend/provider_state.py +++ b/src/govoplan_mail/backend/provider_state.py @@ -16,6 +16,7 @@ from govoplan_mail.backend.db.models import ( MailDeliveryCommand, MailMailboxFolderIndex, MailMailboxMessageIndex, + MailPop3Import, MailServerEndpoint, MailServerProfile, ) @@ -23,6 +24,7 @@ from govoplan_mail.backend.db.models import ( SMTP_PROVIDER_ID = "mail.smtp_delivery" IMAP_PROVIDER_ID = "mail.imap_mailbox" +POP3_PROVIDER_ID = "mail.pop3_legacy_import" _CURRENT_INDEX_WINDOW = timedelta(minutes=30) @@ -38,6 +40,41 @@ def imap_provider_states( return _mail_provider_states(context, protocol="imap") +def pop3_provider_states( + context: ExternalProviderStateContext, +) -> tuple[ExternalProviderRuntimeState, ...]: + if not isinstance(context.session, Session): + raise RuntimeError("Mail provider state requires a database session.") + profiles = _profiles(context) + if not profiles: + return () + profile_ids = tuple(item.id for item in profiles) + endpoints = _endpoints( + context.session, + profile_ids=profile_ids, + protocol="pop3", + ) + endpoints_by_profile: dict[str, list[MailServerEndpoint]] = defaultdict(list) + for endpoint in endpoints: + endpoints_by_profile[endpoint.profile_id].append(endpoint) + metrics = _pop3_metrics( + context.session, + profile_ids=profile_ids, + tenant_id=context.tenant_id, + ) + observed_at = datetime.now(UTC) + return tuple( + _pop3_state( + profile, + endpoints=endpoints_by_profile.get(profile.id, []), + metrics=metrics.get(profile.id, {}), + observed_at=observed_at, + ) + for profile in profiles + if endpoints_by_profile.get(profile.id) + ) + + def _mail_provider_states( context: ExternalProviderStateContext, *, @@ -194,6 +231,37 @@ def _imap_metrics( return result +def _pop3_metrics( + session: Session, + *, + profile_ids: tuple[str, ...], + tenant_id: str | None, +) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = defaultdict(dict) + statement = select( + MailPop3Import.profile_id, + MailPop3Import.deletion_status, + func.count(MailPop3Import.id), + func.max(MailPop3Import.imported_at), + ).where(MailPop3Import.profile_id.in_(profile_ids)) + if tenant_id is not None: + statement = statement.where(MailPop3Import.tenant_id == tenant_id) + rows = session.execute( + statement.group_by( + MailPop3Import.profile_id, + MailPop3Import.deletion_status, + ) + ) + for profile_id, deletion_status, count, last_imported_at in rows: + item = result[str(profile_id)] + item[f"deletion_{deletion_status}"] = int(count) + current = _aware(item.get("last_imported_at")) + candidate = _aware(last_imported_at) + if candidate is not None and (current is None or candidate > current): + item["last_imported_at"] = candidate + return result + + def _smtp_state( profile: MailServerProfile, *, @@ -321,6 +389,71 @@ def _imap_state( ) +def _pop3_state( + profile: MailServerProfile, + *, + endpoints: list[MailServerEndpoint], + metrics: dict[str, Any], + observed_at: datetime, +) -> ExternalProviderRuntimeState: + enabled_endpoints = [ + item + for item in endpoints + if item.is_active and bool((item.config or {}).get("legacy_import_enabled")) + ] + active = bool(profile.is_active) and bool(enabled_endpoints) + failed_deletions = int(metrics.get("deletion_failed", 0)) + unknown_deletions = int(metrics.get("deletion_outcome_unknown", 0)) + last_imported_at = _aware(metrics.get("last_imported_at")) + health = ( + "inactive" + if not active + else "warning" + if failed_deletions or unknown_deletions + else "healthy" + if last_imported_at is not None + else "unknown" + ) + return ExternalProviderRuntimeState( + provider_id=POP3_PROVIDER_ID, + binding_ref=f"mail:profile:{profile.id}:pop3", + authority_mode="governance_overlay", + observed_at=observed_at, + configured=True, + active=active, + health=health, + freshness="not_applicable", + conflict="pending" if unknown_deletions else "clear", + recovery=( + "not_applicable" + if not active + else "attention" + if failed_deletions or unknown_deletions + else "ready" + ), + last_success_at=last_imported_at, + detail=( + "POP3 legacy import is disabled." + if not active + else "POP3 source deletion evidence requires attention." + if failed_deletions or unknown_deletions + else "POP3 legacy import is enabled but has no retained import yet." + if last_imported_at is None + else "POP3 governed import evidence is available." + ), + metrics={ + "active_endpoints": len(enabled_endpoints), + "imports": sum( + int(value) + for key, value in metrics.items() + if key.startswith("deletion_") + ), + "failed_deletions": failed_deletions, + "outcome_unknown_deletions": unknown_deletions, + }, + ) + + def _legacy_configured(profile: MailServerProfile, protocol: str) -> bool: value = profile.smtp_config if protocol == "smtp" else profile.imap_config return isinstance(value, dict) and bool(value) @@ -336,7 +469,9 @@ def _aware(value: object | None) -> datetime | None: __all__ = [ "IMAP_PROVIDER_ID", + "POP3_PROVIDER_ID", "SMTP_PROVIDER_ID", "imap_provider_states", + "pop3_provider_states", "smtp_provider_states", ] diff --git a/src/govoplan_mail/backend/router.py b/src/govoplan_mail/backend/router.py index 0d48385..144dcb5 100644 --- a/src/govoplan_mail/backend/router.py +++ b/src/govoplan_mail/backend/router.py @@ -1,11 +1,12 @@ from __future__ import annotations import dataclasses +import hashlib from types import SimpleNamespace from typing import Any from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlalchemy import func, or_ +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session from govoplan_mail.backend.schemas import ( @@ -43,6 +44,13 @@ from govoplan_mail.backend.schemas import ( MailProfilePolicyResponse, MailProfilePolicyUpdateRequest, MailSettingsDeltaResponse, + MailPop3ImportListResponse, + MailPop3ImportRecordResponse, + MailPop3ImportRequest, + MailPop3ImportResponse, + MailPop3MessagePreviewResponse, + MailPop3PreviewRequest, + MailPop3PreviewResponse, MailServerProfileCreateRequest, MailServerEndpointCreateRequest, MailServerEndpointResponse, @@ -91,7 +99,15 @@ from govoplan_mail.backend.mail_profiles import ( smtp_config_from_profile, update_mail_server_profile, ) -from govoplan_mail.backend.config import ImapConfig, SmtpConfig +from govoplan_mail.backend.config import ImapConfig, Pop3Config, SmtpConfig +from govoplan_mail.backend.db.models import MailPop3Import +from govoplan_mail.backend.pop3_imports import ( + Pop3ImportError, + create_pop3_imports, + list_pop3_imports, + mark_pop3_deletion_result, + pop3_import_payload, +) from govoplan_mail.backend.runtime import get_registry from govoplan_mail.backend.recovery import ( MailRecoveryError, @@ -139,6 +155,14 @@ from govoplan_mail.backend.server_hierarchy import ( ) from govoplan_mail.backend.sending.imap import ImapAppendError, ImapConfigurationError, get_imap_message, list_imap_folders, list_imap_messages, load_imap_mailbox_bootstrap, test_imap_login from govoplan_mail.backend.sending.smtp import test_smtp_login +from govoplan_mail.backend.sending.pop3 import ( + Pop3ConfigurationError, + Pop3ProviderError, + delete_pop3_messages, + download_pop3_messages, + preview_pop3_messages, + test_pop3_login, +) router = APIRouter(prefix="/mail", tags=["mail"]) @@ -1593,6 +1617,8 @@ def create_profile_server( principal: ApiPrincipal = Depends(get_api_principal), session: Session = Depends(get_session), ): + if payload.protocol == "pop3": + _require_scope(principal, "mail:pop3:manage") try: profile = _profile_for_mutation( session, @@ -1652,6 +1678,8 @@ def update_profile_server( profile_id=profile_id, server_id=server_id, ) + if server.protocol == "pop3": + _require_scope(principal, "mail:pop3:manage") update_mail_server_endpoint( session, server=server, @@ -1703,6 +1731,8 @@ def deactivate_profile_server( profile_id=profile_id, server_id=server_id, ) + if server.protocol == "pop3": + _require_scope(principal, "mail:pop3:manage") update_mail_server_endpoint( session, server=server, @@ -2666,6 +2696,359 @@ def test_profile_imap( return MailConnectionTestResponse(ok=False, protocol="imap", message=_safe_error_message(exc), details={"error_type": exc.__class__.__name__}) +def _resolve_profile_pop3_transport( + session: Session, + *, + principal: ApiPrincipal, + profile_id: str, + server_id: str, + credential_id: str | None, +): + profile = _get_profile_for_principal( + session, + principal=principal, + profile_id=profile_id, + require_active=True, + ) + resolved = resolve_mail_transport( + session, + profile=profile, + protocol="pop3", + context=_transport_context_for_principal( + session, + principal=principal, + ), + server_id=server_id, + credential_id=credential_id, + ) + if resolved.server is None or not isinstance(resolved.config, Pop3Config): + raise MailServerHierarchyError("The selected POP3 server is unavailable") + return profile, resolved + + +@router.post( + "/profiles/{profile_id}/test-pop3", + response_model=MailConnectionTestResponse, +) +def test_profile_pop3( + profile_id: str, + server_id: str = Query(...), + credential_id: str | None = Query(default=None), + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "mail:profile:test") + _require_scope(principal, "mail:profile:use") + _require_any_scope(principal, "mail:pop3:manage", "mail:pop3:import") + try: + _profile, resolved = _resolve_profile_pop3_transport( + session, + principal=principal, + profile_id=profile_id, + server_id=server_id, + credential_id=credential_id, + ) + result = test_pop3_login(pop3_config=resolved.config) + return MailConnectionTestResponse( + ok=True, + protocol="pop3", + host=result.host, + port=result.port, + security=result.security, + message="POP3 connection successful.", + details={ + "authenticated": result.authenticated, + "message_count": result.message_count, + "mailbox_size_bytes": result.mailbox_size_bytes, + "legacy_import": True, + }, + ) + except (MailProfileError, MailServerHierarchyError) as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (Pop3ConfigurationError, Pop3ProviderError) as exc: + return MailConnectionTestResponse( + ok=False, + protocol="pop3", + message=_safe_error_message(exc), + details={"error_type": exc.__class__.__name__, "legacy_import": True}, + ) + + +@router.post( + "/profiles/{profile_id}/pop3/preview", + response_model=MailPop3PreviewResponse, +) +def preview_profile_pop3_import( + profile_id: str, + payload: MailPop3PreviewRequest, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "mail:profile:use") + _require_scope(principal, "mail:pop3:import") + try: + _profile, resolved = _resolve_profile_pop3_transport( + session, + principal=principal, + profile_id=profile_id, + server_id=payload.server_id, + credential_id=payload.credential_id, + ) + result = preview_pop3_messages( + pop3_config=resolved.config, + limit=payload.limit, + ) + uidls = [message.uidl for message in result.messages] + imported_uidls = set( + session.scalars( + select(MailPop3Import.provider_uidl).where( + MailPop3Import.tenant_id == principal.tenant_id, + MailPop3Import.profile_id == profile_id, + MailPop3Import.pop3_server_id == resolved.server.id, + MailPop3Import.provider_uidl.in_(uidls), + ) + ) + ) if uidls else set() + return MailPop3PreviewResponse( + profile_id=profile_id, + server_id=resolved.server.id, + transport_revision=resolved.transport_revision, + host=result.host, + port=result.port, + security=result.security, + message_count=result.message_count, + mailbox_size_bytes=result.mailbox_size_bytes, + delete_after_import_allowed=resolved.config.allow_delete_after_import, + messages=[ + MailPop3MessagePreviewResponse( + message_number=message.message_number, + uidl=message.uidl, + subject=message.subject, + from_header=message.from_header, + to_header=message.to_header, + date=message.date, + message_id=message.message_id, + size_bytes=message.size_bytes, + body_preview=message.body_preview, + already_imported=message.uidl in imported_uidls, + ) + for message in result.messages + ], + ) + except (MailProfileError, MailServerHierarchyError) as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except Pop3ConfigurationError as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=str(exc), + ) from exc + except Pop3ProviderError as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=str(exc), + ) from exc + + +@router.post( + "/profiles/{profile_id}/pop3/import", + response_model=MailPop3ImportResponse, +) +def import_profile_pop3_messages( + profile_id: str, + payload: MailPop3ImportRequest, + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "mail:profile:use") + _require_scope(principal, "mail:pop3:import") + if payload.delete_after_import: + _require_scope(principal, "mail:pop3:delete") + try: + _profile, resolved = _resolve_profile_pop3_transport( + session, + principal=principal, + profile_id=profile_id, + server_id=payload.server_id, + credential_id=payload.credential_id, + ) + if resolved.transport_revision != payload.expected_transport_revision: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="The POP3 server or credential selection changed; refresh the preview before importing", + ) + if payload.delete_after_import and not resolved.config.allow_delete_after_import: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Delete-after-import is disabled for the selected POP3 server", + ) + downloaded = download_pop3_messages( + pop3_config=resolved.config, + uidls=payload.uidls, + ) + imported = create_pop3_imports( + session, + tenant_id=principal.tenant_id, + profile_id=profile_id, + pop3_server_id=resolved.server.id, + pop3_credential_id=( + resolved.credential.id if resolved.credential is not None else None + ), + transport_revision=resolved.transport_revision, + messages=downloaded, + user_id=principal.user.id, + deletion_requested=payload.delete_after_import, + ) + aggregate_digest = hashlib.sha256( + "|".join(sorted(item.raw_sha256 for item in downloaded)).encode("ascii") + ).hexdigest() + audit_event( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + action="mail.pop3.imported", + object_type="mail_pop3_import_batch", + object_id=aggregate_digest, + details={ + "profile_id": profile_id, + "server_id": resolved.server.id, + "transport_revision": resolved.transport_revision, + "selected_count": len(downloaded), + "imported_count": len(imported.imported), + "duplicate_count": len(imported.duplicates), + "delete_after_import": payload.delete_after_import, + "content_digest": aggregate_digest, + }, + ) + # The governed local copy and its audit evidence become durable before + # any separately authorized provider deletion is attempted. + session.commit() + + deletion_status = "not_requested" + if payload.delete_after_import: + if not imported.imported: + deletion_status = "skipped_no_new_messages" + else: + new_uidls = [row.provider_uidl for row in imported.imported] + try: + delete_pop3_messages( + pop3_config=resolved.config, + uidls=new_uidls, + ) + deletion_status = "succeeded" + deletion_error = None + except Pop3ProviderError as exc: + deletion_status = ( + "outcome_unknown" if exc.outcome_unknown else "failed" + ) + deletion_error = str(exc) + rows = mark_pop3_deletion_result( + session, + tenant_id=principal.tenant_id, + import_ids=[row.id for row in imported.imported], + status=deletion_status, + error=deletion_error, + ) + # Persist the provider outcome independently of the audit + # projection. If audit insertion is unavailable after the + # irreversible provider operation, the import record still + # retains the result for reconciliation and the pre-effect + # import audit already proves that deletion was requested. + session.commit() + audit_event( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + action="mail.pop3.source_deletion", + object_type="mail_pop3_import_batch", + object_id=aggregate_digest, + details={ + "profile_id": profile_id, + "server_id": resolved.server.id, + "import_count": len(rows), + "status": deletion_status, + }, + ) + session.commit() + + return MailPop3ImportResponse( + imports=[ + MailPop3ImportRecordResponse.model_validate( + pop3_import_payload(row) + ) + for row in imported.imported + ], + duplicate_uidls=sorted(row.provider_uidl for row in imported.duplicates), + deletion_status=deletion_status, + ) + except HTTPException: + session.rollback() + raise + except (MailProfileError, MailServerHierarchyError) as exc: + session.rollback() + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (Pop3ConfigurationError, Pop3ImportError) as exc: + session.rollback() + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=str(exc), + ) from exc + except Pop3ProviderError as exc: + session.rollback() + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=str(exc), + ) from exc + except Exception: + session.rollback() + raise + + +@router.get("/pop3/imports", response_model=MailPop3ImportListResponse) +def get_pop3_imports( + profile_id: str | None = Query(default=None), + limit: int = Query(default=100, ge=1, le=500), + principal: ApiPrincipal = Depends(get_api_principal), + session: Session = Depends(get_session), +): + _require_scope(principal, "mail:pop3:import") + visible_profile_ids = { + profile.id + for profile in list_mail_server_profiles( + session, + tenant_id=principal.tenant_id, + include_inactive=True, + **_profile_actor_kwargs(principal, administrative_visibility=True), + ) + } + if profile_id and profile_id not in visible_profile_ids: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Mail-server profile not found", + ) + rows = list_pop3_imports( + session, + tenant_id=principal.tenant_id, + profile_id=profile_id, + profile_ids=visible_profile_ids, + limit=limit, + ) + return MailPop3ImportListResponse( + imports=[ + MailPop3ImportRecordResponse.model_validate(pop3_import_payload(row)) + for row in rows + ] + ) + + @router.post("/profiles/{profile_id}/list-imap-folders", response_model=MailImapFolderListResponse) def list_profile_imap_folders( profile_id: str, diff --git a/src/govoplan_mail/backend/schemas.py b/src/govoplan_mail/backend/schemas.py index 68bfb53..144735b 100644 --- a/src/govoplan_mail/backend/schemas.py +++ b/src/govoplan_mail/backend/schemas.py @@ -197,7 +197,7 @@ class MailServerEndpointResponse(BaseModel): id: str profile_id: str tenant_id: str | None = None - protocol: Literal["smtp", "imap"] + protocol: Literal["smtp", "imap", "pop3"] name: str config: dict[str, Any] = Field(default_factory=dict) scope_type: MailProfileScope @@ -214,7 +214,7 @@ class MailServerEndpointResponse(BaseModel): class MailServerEndpointCreateRequest(BaseModel): model_config = ConfigDict(extra="forbid") - protocol: Literal["smtp", "imap"] + protocol: Literal["smtp", "imap", "pop3"] name: str = Field(min_length=1, max_length=255) config: dict[str, Any] = Field(default_factory=dict) inherit_to_lower_scopes: bool | None = None @@ -383,7 +383,7 @@ class MailContactCreateResponse(BaseModel): class MailConnectionTestResponse(BaseModel): ok: bool - protocol: Literal["smtp", "imap"] + protocol: Literal["smtp", "imap", "pop3"] host: str | None = None port: int | None = None security: str | None = None @@ -391,6 +391,82 @@ class MailConnectionTestResponse(BaseModel): details: dict[str, Any] = Field(default_factory=dict) +class MailPop3PreviewRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + server_id: str = Field(min_length=1, max_length=36) + credential_id: str | None = Field(default=None, max_length=36) + limit: int = Field(default=50, ge=1, le=100) + + +class MailPop3MessagePreviewResponse(BaseModel): + message_number: int + uidl: str + subject: str | None = None + from_header: str | None = None + to_header: str | None = None + date: str | None = None + message_id: str | None = None + size_bytes: int = 0 + body_preview: str | None = None + already_imported: bool = False + + +class MailPop3PreviewResponse(BaseModel): + profile_id: str + server_id: str + transport_revision: str + host: str + port: int + security: str + message_count: int + mailbox_size_bytes: int + delete_after_import_allowed: bool = False + messages: list[MailPop3MessagePreviewResponse] = Field(default_factory=list) + + +class MailPop3ImportRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + server_id: str = Field(min_length=1, max_length=36) + credential_id: str | None = Field(default=None, max_length=36) + expected_transport_revision: str = Field(min_length=1, max_length=120) + uidls: list[str] = Field(min_length=1, max_length=100) + delete_after_import: bool = False + + +class MailPop3ImportRecordResponse(BaseModel): + id: str + profile_id: str + pop3_server_id: str + transport_revision: str + provider_uidl: str + message_id: str | None = None + subject: str | None = None + from_header: str | None = None + to_header: str | None = None + date: str | None = None + body_preview: str | None = None + size_bytes: int + raw_sha256: str + status: str + imported_at: datetime + deletion_requested: bool = False + deletion_status: str + deletion_attempted_at: datetime | None = None + deletion_error: str | None = None + + +class MailPop3ImportResponse(BaseModel): + imports: list[MailPop3ImportRecordResponse] = Field(default_factory=list) + duplicate_uidls: list[str] = Field(default_factory=list) + deletion_status: str = "not_requested" + + +class MailPop3ImportListResponse(BaseModel): + imports: list[MailPop3ImportRecordResponse] = Field(default_factory=list) + + class MailImapFolderResponse(BaseModel): name: str flags: list[str] = Field(default_factory=list) diff --git a/src/govoplan_mail/backend/sending/pop3.py b/src/govoplan_mail/backend/sending/pop3.py new file mode 100644 index 0000000..c062174 --- /dev/null +++ b/src/govoplan_mail/backend/sending/pop3.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import hashlib +import poplib +import socket +import ssl +from dataclasses import dataclass +from email import policy +from email.message import Message +from email.parser import BytesParser +from typing import Iterable + +from govoplan_core.security.outbound_http import ( + OutboundHttpError, + create_outbound_connection, + validate_outbound_host, +) +from govoplan_mail.backend.config import Pop3Config, TransportSecurity + + +class _OutboundPolicyPOP3(poplib.POP3): + def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def] + return create_outbound_connection( + self.host, + self.port, + timeout=timeout, + label="POP3 legacy import", + ) + + +class _OutboundPolicyPOP3SSL(poplib.POP3_SSL): + def _create_socket(self, timeout: float | None): # type: ignore[no-untyped-def] + sock = create_outbound_connection( + self.host, + self.port, + timeout=timeout, + label="POP3 legacy import", + ) + try: + return self.context.wrap_socket(sock, server_hostname=self.host) + except Exception: + sock.close() + raise + + +class Pop3ConfigurationError(ValueError): + pass + + +class Pop3ProviderError(RuntimeError): + def __init__(self, message: str, *, outcome_unknown: bool = False): + super().__init__(message) + self.outcome_unknown = outcome_unknown + + +@dataclass(frozen=True, slots=True) +class Pop3LoginTestResult: + host: str + port: int + security: str + authenticated: bool + message_count: int + mailbox_size_bytes: int + + +@dataclass(frozen=True, slots=True) +class Pop3MessageSummary: + message_number: int + uidl: str + subject: str | None + from_header: str | None + to_header: str | None + date: str | None + message_id: str | None + size_bytes: int + body_preview: str | None + + +@dataclass(frozen=True, slots=True) +class Pop3PreviewResult: + host: str + port: int + security: str + message_count: int + mailbox_size_bytes: int + messages: tuple[Pop3MessageSummary, ...] + + +@dataclass(frozen=True, slots=True) +class Pop3DownloadedMessage: + message_number: int + uidl: str + raw: bytes + raw_sha256: str + summary: Pop3MessageSummary + + +@dataclass(frozen=True, slots=True) +class Pop3DeletionResult: + deleted_uidls: tuple[str, ...] + + +def _require_pop3_config(config: Pop3Config) -> tuple[str, int]: + if not config.legacy_import_enabled: + raise Pop3ConfigurationError( + "POP3 legacy import is disabled for the selected server" + ) + if not config.host: + raise Pop3ConfigurationError("POP3 host is required") + if not config.port: + raise Pop3ConfigurationError("POP3 port is required") + if not config.username or not config.password: + raise Pop3ConfigurationError("POP3 username and password are required") + return config.host, config.port + + +def _open_pop3(config: Pop3Config) -> poplib.POP3: + host, port = _require_pop3_config(config) + try: + validate_outbound_host(host, port=port, label="POP3 legacy import") + except OutboundHttpError as exc: + raise Pop3ConfigurationError(str(exc)) from exc + + context = ssl.create_default_context() + client: poplib.POP3 | None = None + try: + if config.security == TransportSecurity.TLS: + client = _OutboundPolicyPOP3SSL( + host=host, + port=port, + timeout=config.timeout_seconds, + context=context, + ) + else: + client = _OutboundPolicyPOP3( + host=host, + port=port, + timeout=config.timeout_seconds, + ) + if config.security == TransportSecurity.STARTTLS: + client.stls(context=context) + client.user(config.username) + client.pass_(config.password) + return client + except ssl.SSLError as exc: + _close_without_commit(client) + raise Pop3ProviderError("POP3 TLS negotiation failed") from exc + except poplib.error_proto as exc: + _close_without_commit(client) + raise Pop3ProviderError("POP3 authentication failed") from exc + except (OSError, socket.error) as exc: + _close_without_commit(client) + raise Pop3ProviderError("POP3 connection failed") from exc + except Exception: + _close_without_commit(client) + raise + + +def test_pop3_login(*, pop3_config: Pop3Config) -> Pop3LoginTestResult: + client = _open_pop3(pop3_config) + try: + message_count, mailbox_size = client.stat() + return Pop3LoginTestResult( + host=str(pop3_config.host), + port=int(pop3_config.port or 0), + security=pop3_config.security.value, + authenticated=True, + message_count=int(message_count), + mailbox_size_bytes=int(mailbox_size), + ) + except poplib.error_proto as exc: + raise Pop3ProviderError("POP3 mailbox statistics are unavailable") from exc + finally: + _quit_without_deletions(client) + + +def preview_pop3_messages( + *, + pop3_config: Pop3Config, + limit: int = 50, +) -> Pop3PreviewResult: + clean_limit = max(1, min(int(limit), 100)) + client = _open_pop3(pop3_config) + try: + message_count, mailbox_size = client.stat() + uidls = _uidl_map(client) + sizes = _size_map(client) + selected_numbers = sorted(uidls, reverse=True)[:clean_limit] + messages = tuple( + _preview_message( + client, + message_number=number, + uidl=uidls[number], + size_bytes=sizes.get(number, 0), + body_lines=pop3_config.preview_body_lines, + max_message_bytes=pop3_config.max_message_bytes, + ) + for number in selected_numbers + ) + return Pop3PreviewResult( + host=str(pop3_config.host), + port=int(pop3_config.port or 0), + security=pop3_config.security.value, + message_count=int(message_count), + mailbox_size_bytes=int(mailbox_size), + messages=messages, + ) + except poplib.error_proto as exc: + raise Pop3ProviderError("POP3 message preview failed") from exc + finally: + _quit_without_deletions(client) + + +def download_pop3_messages( + *, + pop3_config: Pop3Config, + uidls: Iterable[str], +) -> tuple[Pop3DownloadedMessage, ...]: + selected_uidls = tuple(dict.fromkeys(_required_uidl(value) for value in uidls)) + if not selected_uidls: + raise Pop3ConfigurationError("Select at least one POP3 message to import") + if len(selected_uidls) > 100: + raise Pop3ConfigurationError("At most 100 POP3 messages can be imported at once") + + client = _open_pop3(pop3_config) + try: + uidl_by_number = _uidl_map(client) + number_by_uidl = {uidl: number for number, uidl in uidl_by_number.items()} + missing = [uidl for uidl in selected_uidls if uidl not in number_by_uidl] + if missing: + raise Pop3ProviderError( + "One or more previewed POP3 messages are no longer available; refresh the preview" + ) + sizes = _size_map(client) + advertised_batch_size = sum( + max(0, int(sizes.get(number_by_uidl[uidl], 0))) + for uidl in selected_uidls + ) + if advertised_batch_size > pop3_config.max_batch_bytes: + raise Pop3ProviderError( + "The selected POP3 messages exceed the configured batch size limit" + ) + downloaded: list[Pop3DownloadedMessage] = [] + downloaded_bytes = 0 + for uidl in selected_uidls: + number = number_by_uidl[uidl] + advertised_size = sizes.get(number, 0) + if advertised_size > pop3_config.max_message_bytes: + raise Pop3ProviderError( + f"POP3 message {uidl} exceeds the configured import size limit" + ) + _response, lines, _octets = client.retr(number) + raw = _message_bytes(lines) + if len(raw) > pop3_config.max_message_bytes: + raise Pop3ProviderError( + f"POP3 message {uidl} exceeds the configured import size limit" + ) + downloaded_bytes += len(raw) + if downloaded_bytes > pop3_config.max_batch_bytes: + raise Pop3ProviderError( + "The selected POP3 messages exceed the configured batch size limit" + ) + summary = _message_summary( + raw, + message_number=number, + uidl=uidl, + size_bytes=len(raw), + ) + downloaded.append( + Pop3DownloadedMessage( + message_number=number, + uidl=uidl, + raw=raw, + raw_sha256=hashlib.sha256(raw).hexdigest(), + summary=summary, + ) + ) + return tuple(downloaded) + except poplib.error_proto as exc: + raise Pop3ProviderError("POP3 message download failed") from exc + finally: + _quit_without_deletions(client) + + +def delete_pop3_messages( + *, + pop3_config: Pop3Config, + uidls: Iterable[str], +) -> Pop3DeletionResult: + selected_uidls = tuple(dict.fromkeys(_required_uidl(value) for value in uidls)) + if not selected_uidls: + return Pop3DeletionResult(deleted_uidls=()) + if not pop3_config.allow_delete_after_import: + raise Pop3ConfigurationError( + "POP3 delete-after-import is disabled for the selected server" + ) + + client = _open_pop3(pop3_config) + quit_started = False + try: + number_by_uidl = { + uidl: number for number, uidl in _uidl_map(client).items() + } + missing = [uidl for uidl in selected_uidls if uidl not in number_by_uidl] + if missing: + raise Pop3ProviderError( + "One or more imported POP3 messages are no longer available for deletion" + ) + for uidl in selected_uidls: + client.dele(number_by_uidl[uidl]) + quit_started = True + client.quit() + return Pop3DeletionResult(deleted_uidls=selected_uidls) + except Pop3ProviderError: + _close_without_commit(client) + raise + except poplib.error_proto as exc: + _close_without_commit(client) + raise Pop3ProviderError( + "POP3 deletion outcome is unknown" if quit_started else "POP3 deletion was rejected", + outcome_unknown=quit_started, + ) from exc + except Exception as exc: + _close_without_commit(client) + raise Pop3ProviderError( + "POP3 deletion outcome is unknown" if quit_started else "POP3 deletion failed", + outcome_unknown=quit_started, + ) from exc + + +def _uidl_map(client: poplib.POP3) -> dict[int, str]: + _response, lines, _octets = client.uidl() + result: dict[int, str] = {} + for raw_line in lines: + parts = bytes(raw_line).decode("utf-8", errors="replace").split(maxsplit=1) + if len(parts) != 2 or not parts[0].isdigit(): + continue + uidl = _required_uidl(parts[1]) + result[int(parts[0])] = uidl + if not result: + raise Pop3ProviderError( + "The POP3 server does not provide stable UIDL identifiers; safe import is unavailable" + ) + return result + + +def _size_map(client: poplib.POP3) -> dict[int, int]: + _response, lines, _octets = client.list() + result: dict[int, int] = {} + for raw_line in lines: + parts = bytes(raw_line).decode("ascii", errors="ignore").split(maxsplit=1) + if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit(): + result[int(parts[0])] = int(parts[1]) + return result + + +def _preview_message( + client: poplib.POP3, + *, + message_number: int, + uidl: str, + size_bytes: int, + body_lines: int, + max_message_bytes: int, +) -> Pop3MessageSummary: + raw: bytes | None = None + try: + _response, lines, _octets = client.top(message_number, body_lines) + raw = _message_bytes(lines) + except (poplib.error_proto, AttributeError): + # TOP is optional. Never use RETR as a preview fallback when the + # advertised message already exceeds the configured download bound. + if size_bytes > max_message_bytes: + return Pop3MessageSummary( + message_number=message_number, + uidl=uidl, + subject=None, + from_header=None, + to_header=None, + date=None, + message_id=None, + size_bytes=size_bytes, + body_preview=None, + ) + _response, lines, _octets = client.retr(message_number) + raw = _message_bytes(lines) + if len(raw) > min(max_message_bytes, 256 * 1024): + return Pop3MessageSummary( + message_number=message_number, + uidl=uidl, + subject=None, + from_header=None, + to_header=None, + date=None, + message_id=None, + size_bytes=size_bytes, + body_preview=None, + ) + return _message_summary( + raw, + message_number=message_number, + uidl=uidl, + size_bytes=size_bytes, + ) + + +def _message_summary( + raw: bytes, + *, + message_number: int, + uidl: str, + size_bytes: int, +) -> Pop3MessageSummary: + message = BytesParser(policy=policy.default).parsebytes(raw) + return Pop3MessageSummary( + message_number=message_number, + uidl=uidl, + subject=_header(message, "Subject"), + from_header=_header(message, "From"), + to_header=_header(message, "To"), + date=_header(message, "Date"), + message_id=_header(message, "Message-ID"), + size_bytes=max(0, int(size_bytes)), + body_preview=_body_preview(message), + ) + + +def _header(message: Message, name: str) -> str | None: + value = message.get(name) + if value is None: + return None + text = str(value).strip() + return text[:2_000] or None + + +def _body_preview(message: Message) -> str | None: + body = message.get_body(preferencelist=("plain",)) if message.is_multipart() else message + if body is None: + return None + try: + text = body.get_content() + except Exception: + payload = body.get_payload(decode=True) + text = payload.decode("utf-8", errors="replace") if isinstance(payload, bytes) else str(payload or "") + normalized = " ".join(str(text).split()) + return normalized[:500] or None + + +def _message_bytes(lines: Iterable[bytes]) -> bytes: + return b"\r\n".join(bytes(line) for line in lines) + b"\r\n" + + +def _required_uidl(value: object) -> str: + clean = str(value or "").strip() + if not clean or len(clean) > 500 or any(char.isspace() for char in clean): + raise Pop3ConfigurationError("POP3 UIDL must be a non-empty token") + return clean + + +def _quit_without_deletions(client: poplib.POP3) -> None: + try: + client.quit() + except Exception: + _close_without_commit(client) + + +def _close_without_commit(client: poplib.POP3 | None) -> None: + if client is None: + return + try: + client.rset() + except Exception: + pass + try: + client.close() + except Exception: + pass + + +__all__ = [ + "Pop3ConfigurationError", + "Pop3DeletionResult", + "Pop3DownloadedMessage", + "Pop3LoginTestResult", + "Pop3MessageSummary", + "Pop3PreviewResult", + "Pop3ProviderError", + "delete_pop3_messages", + "download_pop3_messages", + "preview_pop3_messages", + "test_pop3_login", +] diff --git a/src/govoplan_mail/backend/server_hierarchy.py b/src/govoplan_mail/backend/server_hierarchy.py index ba88b4b..e194286 100644 --- a/src/govoplan_mail/backend/server_hierarchy.py +++ b/src/govoplan_mail/backend/server_hierarchy.py @@ -23,6 +23,8 @@ from govoplan_core.security.secrets import decrypt_secret from govoplan_mail.backend.config import ( ImapConfig, ImapServerConfig, + Pop3Config, + Pop3ServerConfig, SmtpConfig, SmtpServerConfig, ) @@ -34,7 +36,7 @@ from govoplan_mail.backend.db.models import ( ) -MAIL_SERVER_PROTOCOLS = frozenset({"smtp", "imap"}) +MAIL_SERVER_PROTOCOLS = frozenset({"smtp", "imap", "pop3"}) class MailServerHierarchyError(RuntimeError): @@ -68,7 +70,7 @@ class ResolvedMailTransport: profile: MailServerProfile server: MailServerEndpoint | None credential: CredentialEnvelope | None - config: SmtpConfig | ImapConfig + config: SmtpConfig | ImapConfig | Pop3Config transport_revision: str @@ -951,6 +953,10 @@ def resolve_mail_transport( ) ) if server is None: + if clean_protocol == "pop3": + raise MailServerHierarchyError( + "The selected Mail profile has no active POP3 legacy-import server" + ) return _legacy_resolved_transport(profile, clean_protocol) binding, credential = _selected_server_credential( session, @@ -978,16 +984,17 @@ def resolve_mail_transport( if clean_protocol == "smtp": payload["username"] = profile.smtp_username payload["password"] = decrypt_secret(profile.smtp_password_encrypted) - else: + elif clean_protocol == "imap": payload["username"] = profile.imap_username payload["password"] = decrypt_secret(profile.imap_password_encrypted) - config: SmtpConfig | ImapConfig + config: SmtpConfig | ImapConfig | Pop3Config try: - config = ( - SmtpConfig.model_validate(payload) - if clean_protocol == "smtp" - else ImapConfig.model_validate(payload) - ) + if clean_protocol == "smtp": + config = SmtpConfig.model_validate(payload) + elif clean_protocol == "imap": + config = ImapConfig.model_validate(payload) + else: + config = Pop3Config.model_validate(payload) except Exception as exc: raise MailServerHierarchyError( f"The selected {clean_protocol.upper()} server configuration is invalid" @@ -1030,6 +1037,14 @@ def select_mail_transport( ) ) if server is None: + if clean_protocol == "pop3": + return SelectedMailTransport( + profile=profile, + server=None, + credential=None, + available=False, + transport_revision="unconfigured", + ) legacy_config = ( profile.smtp_config if clean_protocol == "smtp" @@ -1302,11 +1317,12 @@ def _validated_server_config( payload.pop("password", None) payload.pop("enabled", None) try: - model = ( - SmtpServerConfig.model_validate(payload) - if protocol == "smtp" - else ImapServerConfig.model_validate(payload) - ) + if protocol == "smtp": + model = SmtpServerConfig.model_validate(payload) + elif protocol == "imap": + model = ImapServerConfig.model_validate(payload) + else: + model = Pop3ServerConfig.model_validate(payload) except Exception as exc: raise MailServerHierarchyError( f"Invalid {protocol.upper()} server configuration" @@ -1347,7 +1363,7 @@ def _server_scope( def _normalize_protocol(value: str) -> str: clean = str(value or "").strip().casefold() if clean not in MAIL_SERVER_PROTOCOLS: - raise MailServerHierarchyError("Mail server protocol must be smtp or imap") + raise MailServerHierarchyError("Mail server protocol must be smtp, imap or pop3") return clean diff --git a/tests/test_dsar_provider.py b/tests/test_dsar_provider.py index dc2c730..a392896 100644 --- a/tests/test_dsar_provider.py +++ b/tests/test_dsar_provider.py @@ -22,6 +22,8 @@ from govoplan_mail.backend.db.models import ( MailDeliveryCommand, MailDeliveryReconciliation, MailMailboxMessageIndex, + MailPop3Import, + MailServerEndpoint, MailServerProfile, ) from govoplan_mail.backend.dsar_provider import MAIL_DSAR_CAPABILITY, MailDsarProvider @@ -71,7 +73,9 @@ class MailDsarProviderTests(unittest.TestCase): ChangeSequenceEntry.__table__, DataSubjectRequest.__table__, MailServerProfile.__table__, + MailServerEndpoint.__table__, MailMailboxMessageIndex.__table__, + MailPop3Import.__table__, MailDeliveryCommand.__table__, MailDeliveryAttempt.__table__, MailDeliveryReconciliation.__table__, @@ -160,6 +164,39 @@ class MailDsarProviderTests(unittest.TestCase): to_header="subject@example.test", indexed_at=now, ) + pop3_server = MailServerEndpoint( + id="pop3-server-subject", + profile_id=profile.id, + tenant_id="tenant-1", + protocol="pop3", + name="Legacy POP3", + config={"host": "pop3-secret-do-not-export"}, + scope_type="tenant", + scope_id="tenant-1", + ) + pop3_import = MailPop3Import( + id="pop3-import-subject", + tenant_id="tenant-1", + profile_id=profile.id, + pop3_server_id=pop3_server.id, + pop3_credential_id="credential-secret-do-not-export", + transport_revision="pop3-revision-secret-do-not-export", + provider_uidl="provider-uidl-secret-do-not-export", + fingerprint="e" * 64, + raw_sha256="f" * 64, + raw_message_encrypted="pop3-message-cipher-do-not-export", + message_id="pop3-message-id", + subject="Imported subject notice", + from_header="Legacy office ", + to_header="Subject Person ", + date="2026-08-19", + body_preview="Imported message preview for the subject", + size_bytes=84, + status="pending_review", + imported_at=now, + deletion_requested=False, + deletion_status="not_requested", + ) command = MailDeliveryCommand( id="command-subject", tenant_id="tenant-1", @@ -230,6 +267,8 @@ class MailDsarProviderTests(unittest.TestCase): unrelated, tenant_two_profile, tenant_two, + pop3_server, + pop3_import, command, attempt, reconciliation, @@ -264,6 +303,7 @@ class MailDsarProviderTests(unittest.TestCase): "mail_delivery_attempt", "mail_delivery_reconciliation", "mail_bounce_observation", + "mail_pop3_import", }.issubset({r.resource_type for r in records}) ) serialized = repr([record.to_dict() for record in records]) @@ -295,6 +335,11 @@ class MailDsarProviderTests(unittest.TestCase): "original-id-do-not-export", "bounce-diagnostic-do-not-export", "bounce-evidence-do-not-export", + "pop3-secret-do-not-export", + "credential-secret-do-not-export", + "pop3-revision-secret-do-not-export", + "provider-uidl-secret-do-not-export", + "pop3-message-cipher-do-not-export", ): self.assertNotIn(hidden, serialized) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 494e9f1..8f316b8 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -53,7 +53,22 @@ class MailManifestTests(unittest.TestCase): permissions = {permission.scope for permission in manifest.permissions} self.assertIn("mail:profile:write_own", permissions) self.assertIn("mail:secret:manage_own", permissions) + self.assertTrue( + {"mail:pop3:manage", "mail:pop3:import", "mail:pop3:delete"}.issubset( + permissions + ) + ) roles = {template.slug: template for template in manifest.role_templates} + self.assertIn("mail:pop3:delete", roles["mail_profile_admin"].permissions) + self.assertEqual( + set(roles["mail_legacy_import_operator"].permissions), + { + "mail:profile:read", + "mail:profile:use", + "mail:profile:test", + "mail:pop3:import", + }, + ) self.assertEqual( set(roles["mail_profile_self_service"].permissions), { @@ -73,8 +88,27 @@ class MailManifestTests(unittest.TestCase): "mail.reference.credentials-egress-retirement", "mail.reference.campaign-delivery-contract", "mail.address-book-integration", + "mail.workflow.legacy-pop3-import", }.issubset(topics) ) + pop3_topic = topics["mail.workflow.legacy-pop3-import"] + self.assertEqual(("user", "admin"), pop3_topic.documentation_types) + self.assertIn("mail:pop3:import", pop3_topic.conditions[0].any_scopes) + + pop3_provider = next( + item + for item in manifest.external_providers + if item.id == "mail.pop3_legacy_import" + ) + self.assertIn("delete", pop3_provider.operations) + self.assertTrue(pop3_provider.behavior.outcome_unknown_supported) + self.assertIn("mail.pop3.source_deletion", pop3_provider.behavior.audit_event_types) + self.assertTrue( + any( + route.path == "/mail/legacy-import" + for route in manifest.frontend.routes # type: ignore[union-attr] + ) + ) ownership = topics["mail.profile-ownership-and-consumers"] self.assertEqual(ownership.metadata["kind"], "reference") self.assertEqual(ownership.metadata["route"], "/settings?section=mail-profiles") diff --git a/tests/test_pop3_imports.py b/tests/test_pop3_imports.py new file mode 100644 index 0000000..f8b697d --- /dev/null +++ b/tests/test_pop3_imports.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import base64 +from datetime import UTC, datetime +import poplib +import ssl +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_access.backend.db.models import Account, User +from govoplan_core.auth import ApiPrincipal +from govoplan_core.core.access import PrincipalRef +from govoplan_core.db.base import Base +from govoplan_core.security.secrets import decrypt_secret +from govoplan_mail.backend.config import Pop3Config, TransportSecurity +from govoplan_mail.backend.db.models import ( + MailPop3Import, + MailServerEndpoint, + MailServerProfile, +) +from govoplan_mail.backend.pop3_imports import ( + Pop3ImportResult, + create_pop3_imports, + list_pop3_imports, +) +from govoplan_mail.backend.router import import_profile_pop3_messages +from govoplan_mail.backend.schemas import MailPop3ImportRequest +from govoplan_mail.backend.sending.pop3 import ( + Pop3ConfigurationError, + Pop3DownloadedMessage, + Pop3MessageSummary, + Pop3ProviderError, + _open_pop3, + delete_pop3_messages, + download_pop3_messages, + preview_pop3_messages, +) + + +_RAW = ( + b"Subject: Legacy notice\r\n" + b"From: Office \r\n" + b"To: Subject \r\n" + b"Message-ID: \r\n" + b"\r\n" + b"A bounded legacy message.\r\n" +) + + +class _Pop3Client: + def __init__(self, *, quit_error: Exception | None = None) -> None: + self.deletions: list[int] = [] + self.quit_calls = 0 + self.rset_calls = 0 + self.close_calls = 0 + self.quit_error = quit_error + + def stat(self): + return 1, len(_RAW) + + def uidl(self): + return b"+OK", [b"1 uid-1"], 1 + + def list(self): + return b"+OK", [f"1 {len(_RAW)}".encode("ascii")], 1 + + def top(self, _number, _lines): + return b"+OK", _RAW.rstrip(b"\r\n").split(b"\r\n"), len(_RAW) + + def retr(self, _number): + return b"+OK", _RAW.rstrip(b"\r\n").split(b"\r\n"), len(_RAW) + + def dele(self, number): + self.deletions.append(number) + + def quit(self): + self.quit_calls += 1 + if self.quit_error is not None: + raise self.quit_error + return b"+OK" + + def rset(self): + self.rset_calls += 1 + + def close(self): + self.close_calls += 1 + + +def _config(**changes) -> Pop3Config: + values = { + "host": "pop3.example.test", + "security": "tls", + "username": "legacy-user", + "password": "legacy-password", + "legacy_import_enabled": True, + } + values.update(changes) + return Pop3Config.model_validate(values) + + +def _download(uidl: str = "uid-1") -> Pop3DownloadedMessage: + summary = Pop3MessageSummary( + message_number=1, + uidl=uidl, + subject="Legacy notice", + from_header="Office ", + to_header="Subject ", + date="Sat, 22 Aug 2026 10:00:00 +0200", + message_id="", + size_bytes=len(_RAW), + body_preview="A bounded legacy message.", + ) + import hashlib + + return Pop3DownloadedMessage( + message_number=1, + uidl=uidl, + raw=_RAW, + raw_sha256=hashlib.sha256(_RAW).hexdigest(), + summary=summary, + ) + + +class Pop3TransportTests(unittest.TestCase): + def test_legacy_import_is_disabled_until_explicitly_enabled(self) -> None: + with self.assertRaisesRegex(Pop3ConfigurationError, "disabled"): + preview_pop3_messages( + pop3_config=_config(legacy_import_enabled=False), + limit=10, + ) + with self.assertRaisesRegex(ValueError, "batch size limit"): + _config(max_message_bytes=2 * 1024 * 1024, max_batch_bytes=1024 * 1024) + + def test_preview_and_download_are_non_destructive(self) -> None: + preview_client = _Pop3Client() + with patch( + "govoplan_mail.backend.sending.pop3._open_pop3", + return_value=preview_client, + ): + preview = preview_pop3_messages(pop3_config=_config(), limit=10) + + self.assertEqual(["uid-1"], [item.uidl for item in preview.messages]) + self.assertEqual([], preview_client.deletions) + self.assertEqual(1, preview_client.quit_calls) + + download_client = _Pop3Client() + with patch( + "govoplan_mail.backend.sending.pop3._open_pop3", + return_value=download_client, + ): + downloaded = download_pop3_messages( + pop3_config=_config(), uidls=("uid-1",) + ) + + self.assertEqual(_RAW, downloaded[0].raw) + self.assertEqual([], download_client.deletions) + self.assertEqual(1, download_client.quit_calls) + + def test_source_deletion_needs_policy_and_commits_with_quit(self) -> None: + with self.assertRaisesRegex(Pop3ConfigurationError, "disabled"): + delete_pop3_messages(pop3_config=_config(), uidls=("uid-1",)) + + client = _Pop3Client() + with patch( + "govoplan_mail.backend.sending.pop3._open_pop3", return_value=client + ): + result = delete_pop3_messages( + pop3_config=_config(allow_delete_after_import=True), + uidls=("uid-1",), + ) + + self.assertEqual(("uid-1",), result.deleted_uidls) + self.assertEqual([1], client.deletions) + self.assertEqual(1, client.quit_calls) + + def test_quit_failure_marks_deletion_outcome_unknown(self) -> None: + client = _Pop3Client(quit_error=poplib.error_proto("connection lost")) + with ( + patch( + "govoplan_mail.backend.sending.pop3._open_pop3", + return_value=client, + ), + self.assertRaises(Pop3ProviderError) as captured, + ): + delete_pop3_messages( + pop3_config=_config(allow_delete_after_import=True), + uidls=("uid-1",), + ) + + self.assertTrue(captured.exception.outcome_unknown) + self.assertEqual([1], client.deletions) + + def test_tls_and_authentication_failures_are_sanitized(self) -> None: + with ( + patch( + "govoplan_mail.backend.sending.pop3.validate_outbound_host" + ), + patch( + "govoplan_mail.backend.sending.pop3._OutboundPolicyPOP3SSL", + side_effect=ssl.SSLError("private TLS detail"), + ), + self.assertRaisesRegex(Pop3ProviderError, "TLS negotiation failed"), + ): + _open_pop3(_config()) + + auth_client = _Pop3Client() + auth_client.user = lambda _value: None # type: ignore[attr-defined] + auth_client.pass_ = lambda _value: (_ for _ in ()).throw( # type: ignore[attr-defined] + poplib.error_proto("private auth detail") + ) + with ( + patch( + "govoplan_mail.backend.sending.pop3.validate_outbound_host" + ), + patch( + "govoplan_mail.backend.sending.pop3._OutboundPolicyPOP3", + return_value=auth_client, + ), + self.assertRaisesRegex(Pop3ProviderError, "authentication failed"), + ): + _open_pop3(_config(security=TransportSecurity.PLAIN)) + + +class Pop3PersistenceTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True) + Base.metadata.create_all( + self.engine, + tables=( + Account.__table__, + User.__table__, + MailServerProfile.__table__, + MailServerEndpoint.__table__, + MailPop3Import.__table__, + ), + ) + self.session = sessionmaker(bind=self.engine, expire_on_commit=False)() + self.profile = MailServerProfile( + id="profile-1", + tenant_id="tenant-1", + scope_type="tenant", + scope_id="tenant-1", + name="Legacy source", + slug="legacy-source", + smtp_config={}, + ) + self.server = MailServerEndpoint( + id="server-1", + profile_id=self.profile.id, + tenant_id="tenant-1", + protocol="pop3", + name="Legacy POP3", + config={"legacy_import_enabled": True}, + scope_type="tenant", + scope_id="tenant-1", + transport_revision="revision-1", + ) + self.session.add_all((self.profile, self.server)) + self.session.commit() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_import_is_encrypted_and_duplicate_uidl_is_reused(self) -> None: + first = create_pop3_imports( + self.session, + tenant_id="tenant-1", + profile_id=self.profile.id, + pop3_server_id=self.server.id, + pop3_credential_id=None, + transport_revision="revision-1", + messages=(_download(),), + user_id=None, + deletion_requested=False, + ) + self.session.commit() + + self.assertEqual(1, len(first.imported)) + encrypted = first.imported[0].raw_message_encrypted + self.assertNotIn("Legacy notice", encrypted) + self.assertEqual( + _RAW, + base64.b64decode(decrypt_secret(encrypted) or ""), + ) + self.assertEqual("not_requested", first.imported[0].deletion_status) + + repeated = create_pop3_imports( + self.session, + tenant_id="tenant-1", + profile_id=self.profile.id, + pop3_server_id=self.server.id, + pop3_credential_id=None, + transport_revision="revision-1", + messages=(_download(),), + user_id=None, + deletion_requested=False, + ) + + self.assertEqual((), repeated.imported) + self.assertEqual((first.imported[0].id,), tuple(row.id for row in repeated.duplicates)) + self.assertEqual( + (first.imported[0].id,), + tuple( + row.id + for row in list_pop3_imports( + self.session, + tenant_id="tenant-1", + profile_ids=(self.profile.id,), + ) + ), + ) + self.assertEqual( + (), + list_pop3_imports( + self.session, + tenant_id="tenant-1", + profile_ids=("unrelated-profile",), + ), + ) + + +class _RouteSession: + def __init__(self, events: list[str]) -> None: + self.events = events + + def commit(self) -> None: + self.events.append("commit") + + def rollback(self) -> None: + self.events.append("rollback") + + +class Pop3ImportRouteTests(unittest.TestCase): + def test_local_import_and_audit_commit_before_source_deletion(self) -> None: + events: list[str] = [] + now = datetime.now(UTC) + row = SimpleNamespace( + id="import-1", + profile_id="profile-1", + pop3_server_id="server-1", + transport_revision="revision-1", + provider_uidl="uid-1", + message_id="", + subject="Legacy notice", + from_header="office@example.test", + to_header="subject@example.test", + date="2026-08-22", + body_preview="A bounded legacy message.", + size_bytes=len(_RAW), + raw_sha256=_download().raw_sha256, + status="pending_review", + imported_at=now, + deletion_requested=True, + deletion_status="pending", + deletion_attempted_at=None, + deletion_error=None, + ) + resolved = SimpleNamespace( + config=_config(allow_delete_after_import=True), + server=SimpleNamespace(id="server-1"), + credential=None, + transport_revision="revision-1", + ) + principal = ApiPrincipal( + principal=PrincipalRef( + account_id="account-1", + membership_id="user-1", + tenant_id="tenant-1", + scopes=frozenset( + { + "mail:profile:use", + "mail:pop3:import", + "mail:pop3:delete", + } + ), + ), + account=SimpleNamespace(id="account-1"), + user=SimpleNamespace(id="user-1"), + ) + payload = MailPop3ImportRequest( + server_id="server-1", + expected_transport_revision="revision-1", + uidls=["uid-1"], + delete_after_import=True, + ) + + def audit(*_args, **_kwargs) -> None: + events.append("audit") + + def delete(**_kwargs): + self.assertEqual(["audit", "commit"], events) + events.append("delete") + + def mark(*_args, **_kwargs): + events.append("mark") + row.deletion_status = "succeeded" + row.deletion_attempted_at = now + return (row,) + + with ( + patch( + "govoplan_mail.backend.router._resolve_profile_pop3_transport", + return_value=(SimpleNamespace(id="profile-1"), resolved), + ), + patch( + "govoplan_mail.backend.router.download_pop3_messages", + return_value=(_download(),), + ), + patch( + "govoplan_mail.backend.router.create_pop3_imports", + return_value=Pop3ImportResult(imported=(row,), duplicates=()), + ), + patch("govoplan_mail.backend.router.audit_event", side_effect=audit), + patch("govoplan_mail.backend.router.delete_pop3_messages", side_effect=delete), + patch( + "govoplan_mail.backend.router.mark_pop3_deletion_result", + side_effect=mark, + ), + ): + result = import_profile_pop3_messages( + "profile-1", + payload, + principal=principal, + session=_RouteSession(events), # type: ignore[arg-type] + ) + + self.assertEqual("succeeded", result.deletion_status) + self.assertEqual( + ["audit", "commit", "delete", "mark", "commit", "audit", "commit"], + events, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_provider_state.py b/tests/test_provider_state.py index 9b13809..4288fe7 100644 --- a/tests/test_provider_state.py +++ b/tests/test_provider_state.py @@ -14,14 +14,17 @@ from govoplan_mail.backend.db.models import ( MailDeliveryCommand, MailMailboxFolderIndex, MailMailboxMessageIndex, + MailPop3Import, MailServerEndpoint, MailServerProfile, ) from govoplan_mail.backend.manifest import manifest from govoplan_mail.backend.provider_state import ( IMAP_PROVIDER_ID, + POP3_PROVIDER_ID, SMTP_PROVIDER_ID, imap_provider_states, + pop3_provider_states, smtp_provider_states, ) @@ -38,6 +41,7 @@ class MailProviderStateTests(unittest.TestCase): MailMailboxFolderIndex.__table__, MailMailboxMessageIndex.__table__, MailBounceSource.__table__, + MailPop3Import.__table__, ), ) self.session = sessionmaker(bind=self.engine, expire_on_commit=False)() @@ -120,17 +124,89 @@ class MailProviderStateTests(unittest.TestCase): ), ) self.assertEqual( - {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID}, + {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID, POP3_PROVIDER_ID}, {item.id for item in manifest.external_providers}, ) self.assertEqual( - {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID}, + {SMTP_PROVIDER_ID, IMAP_PROVIDER_ID, POP3_PROVIDER_ID}, { item.provider_id for item in manifest.external_provider_state_providers }, ) + def test_pop3_state_is_disabled_by_default_and_projects_deletion_evidence(self) -> None: + endpoint = MailServerEndpoint( + id="pop3-server-1", + profile_id=self.profile.id, + tenant_id="tenant-1", + protocol="pop3", + name="Legacy POP3", + config={"host": "pop3.example.test", "legacy_import_enabled": False}, + scope_type="tenant", + scope_id="tenant-1", + is_active=True, + ) + self.session.add(endpoint) + self.session.flush() + + disabled = pop3_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + self.assertFalse(disabled.active) + self.assertEqual("inactive", disabled.health) + + endpoint.config = { + "host": "pop3.example.test", + "legacy_import_enabled": True, + } + self.session.add( + MailPop3Import( + id="pop3-import-1", + tenant_id="tenant-1", + profile_id=self.profile.id, + pop3_server_id=endpoint.id, + transport_revision=endpoint.transport_revision, + provider_uidl="uid-1", + fingerprint="a" * 64, + raw_sha256="b" * 64, + raw_message_encrypted="ciphertext-do-not-project", + size_bytes=42, + imported_at=datetime.now(UTC), + deletion_requested=True, + deletion_status="outcome_unknown", + ) + ) + self.session.add( + MailPop3Import( + id="pop3-import-tenant-2", + tenant_id="tenant-2", + profile_id=self.profile.id, + pop3_server_id=endpoint.id, + transport_revision=endpoint.transport_revision, + provider_uidl="uid-tenant-2", + fingerprint="c" * 64, + raw_sha256="d" * 64, + raw_message_encrypted="other-tenant-ciphertext", + size_bytes=42, + imported_at=datetime.now(UTC), + deletion_requested=True, + deletion_status="failed", + ) + ) + self.session.flush() + + state = pop3_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + self.assertTrue(state.active) + self.assertEqual("warning", state.health) + self.assertEqual("pending", state.conflict) + self.assertEqual(1, state.metrics["outcome_unknown_deletions"]) + self.assertEqual(0, state.metrics["failed_deletions"]) + self.assertNotIn("pop3.example.test", str(state.to_dict())) + self.assertNotIn("ciphertext-do-not-project", str(state.to_dict())) + if __name__ == "__main__": unittest.main() diff --git a/webui/package-lock.json b/webui/package-lock.json index a19c05b..370dd77 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -1,12 +1,12 @@ { "name": "@govoplan/mail-webui", - "version": "0.1.18", + "version": "0.1.20", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govoplan/mail-webui", - "version": "0.1.18", + "version": "0.1.20", "devDependencies": { "typescript": "^5.7.2" }, diff --git a/webui/package.json b/webui/package.json index a14433b..7aa6ca1 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/mail-webui", - "version": "0.1.19", + "version": "0.1.20", "private": true, "type": "module", "main": "src/index.ts", diff --git a/webui/scripts/test-interface-pattern-language.mjs b/webui/scripts/test-interface-pattern-language.mjs index 5880fde..17f94bd 100644 --- a/webui/scripts/test-interface-pattern-language.mjs +++ b/webui/scripts/test-interface-pattern-language.mjs @@ -9,6 +9,7 @@ function read(relativePath) { const profiles = read("../src/features/mail/MailProfileManagement.tsx"); const mailbox = read("../src/features/mail/MailboxPage.tsx"); const bounces = read("../src/features/mail/MailBouncePage.tsx"); +const legacyImport = read("../src/features/mail/MailLegacyImportPage.tsx"); const moduleSource = read("../src/module.ts"); const styles = read("../src/styles/mail-profiles.css"); const migration = read("../../docs/INTERFACE_PATTERN_MIGRATION.md"); @@ -40,8 +41,18 @@ assert.match(bounces, /topicId: "mail\.bounce-processing"/); assert.match(bounces, / void runImport\(\)\}/); +assert.match(legacyImport, /legacy_import_enabled: false[\s\S]*is_active: false[\s\S]*createMailServerCredential[\s\S]*updateMailServerEndpoint/); +assert.match(legacyImport, /expected_transport_revision: preview\.transport_revision/); + +assert.doesNotMatch(`${profiles}\n${mailbox}\n${bounces}\n${legacyImport}`, /window\.(?:alert|confirm)\(/); +assert.doesNotMatch(`${profiles}\n${mailbox}\n${bounces}\n${legacyImport}\n${moduleSource}`, /@govoplan\/(?:campaign|files|docs|calendar)-webui|govoplan_(?:campaign|files|docs|calendar)/); assert.match(moduleSource, /"mail\.profiles"/); assert.match(styles, /@media \(max-width: 900px\)[\s\S]*\.mail-profile-transport-summary[\s\S]*grid-template-columns: 1fr/); assert.match(styles, /@media \(max-width: 1280px\)[\s\S]*\.mailbox-shell\.file-manager-shell[\s\S]*grid-template-columns:/); diff --git a/webui/src/api/mail.ts b/webui/src/api/mail.ts index 22c778c..2d61390 100644 --- a/webui/src/api/mail.ts +++ b/webui/src/api/mail.ts @@ -296,7 +296,7 @@ export async function createMailServerProfile(settings: ApiSettings, payload: Ma export type MailServerProfileUpdatePayload = Partial & { clear_imap?: boolean }; export type MailServerEndpointPayload = { - protocol: "smtp" | "imap"; + protocol: "smtp" | "imap" | "pop3"; name: string; config: Record; inherit_to_lower_scopes?: boolean | null; @@ -304,6 +304,77 @@ export type MailServerEndpointPayload = { is_active?: boolean; }; +export type MailPop3ServerConfig = { + host?: string | null; + port?: number | null; + security?: MailSecurity | string; + timeout_seconds?: number; + max_message_bytes?: number; + max_batch_bytes?: number; + preview_body_lines?: number; + legacy_import_enabled?: boolean; + allow_delete_after_import?: boolean; +}; + +export type MailPop3ServerEndpoint = Omit & { + protocol: "pop3"; + config: MailPop3ServerConfig; +}; + +export type MailPop3MessagePreview = { + message_number: number; + uidl: string; + subject?: string | null; + from_header?: string | null; + to_header?: string | null; + date?: string | null; + message_id?: string | null; + size_bytes: number; + body_preview?: string | null; + already_imported: boolean; +}; + +export type MailPop3PreviewResponse = { + profile_id: string; + server_id: string; + transport_revision: string; + host: string; + port: number; + security: string; + message_count: number; + mailbox_size_bytes: number; + delete_after_import_allowed: boolean; + messages: MailPop3MessagePreview[]; +}; + +export type MailPop3ImportRecord = { + id: string; + profile_id: string; + pop3_server_id: string; + transport_revision: string; + provider_uidl: string; + message_id?: string | null; + subject?: string | null; + from_header?: string | null; + to_header?: string | null; + date?: string | null; + body_preview?: string | null; + size_bytes: number; + raw_sha256: string; + status: string; + imported_at: string; + deletion_requested: boolean; + deletion_status: string; + deletion_attempted_at?: string | null; + deletion_error?: string | null; +}; + +export type MailPop3ImportResponse = { + imports: MailPop3ImportRecord[]; + duplicate_uidls: string[]; + deletion_status: string; +}; + export type MailCredentialCreatePayload = { name: string; description?: string | null; @@ -503,6 +574,63 @@ export async function testMailProfileImap( ); } +export async function testMailProfilePop3( + settings: ApiSettings, + profileId: string, + serverId: string, + credentialId?: string | null +): Promise { + return apiPost( + settings, + apiPath(`/api/v1/mail/profiles/${encodeURIComponent(profileId)}/test-pop3`, { + server_id: serverId, + credential_id: credentialId + }) + ); +} + +export async function previewMailProfilePop3( + settings: ApiSettings, + profileId: string, + payload: { server_id: string; credential_id?: string | null; limit?: number } +): Promise { + return apiPostJson( + settings, + `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/pop3/preview`, + payload + ); +} + +export async function importMailProfilePop3( + settings: ApiSettings, + profileId: string, + payload: { + server_id: string; + credential_id?: string | null; + expected_transport_revision: string; + uidls: string[]; + delete_after_import?: boolean; + } +): Promise { + return apiPostJson( + settings, + `/api/v1/mail/profiles/${encodeURIComponent(profileId)}/pop3/import`, + payload + ); +} + +export async function listMailPop3Imports( + settings: ApiSettings, + profileId?: string | null, + limit = 100 +): Promise { + const response = await apiFetch<{ imports: MailPop3ImportRecord[] }>( + settings, + apiPath("/api/v1/mail/pop3/imports", { profile_id: profileId, limit }) + ); + return response.imports; +} + export async function listMailProfileImapFolders( settings: ApiSettings, profileId: string, diff --git a/webui/src/features/mail/MailLegacyImportPage.tsx b/webui/src/features/mail/MailLegacyImportPage.tsx new file mode 100644 index 0000000..0d1c99e --- /dev/null +++ b/webui/src/features/mail/MailLegacyImportPage.tsx @@ -0,0 +1,610 @@ +import { useEffect, useMemo, useState } from "react"; +import { ArrowLeft, Pencil, Plus, ShieldCheck } from "lucide-react"; +import { + ActionBlockerHint, + Button, + Card, + ConfirmDialog, + ContentGrid, + DataGrid, + Dialog, + DocumentationHelpLink, + FormField, + FormGrid, + PageActionBar, + PageLayout, + SelectionList, + SelectionListItem, + SelectionListItemContent, + StatusBadge, + TableActionGroup, + ToggleSwitch, + adminErrorMessage, + formatDateTime, + hasScope, + useGuardedNavigate, + type ApiSettings, + type AuthInfo, + type DataGridColumn +} from "@govoplan/core-webui"; +import { + createMailServerCredential, + createMailServerEndpoint, + importMailProfilePop3, + listMailPop3Imports, + listMailServerProfiles, + previewMailProfilePop3, + testMailProfilePop3, + updateMailServerCredential, + updateMailServerEndpoint, + type MailCredentialEnvelope, + type MailPop3ImportRecord, + type MailPop3MessagePreview, + type MailPop3PreviewResponse, + type MailPop3ServerEndpoint, + type MailServerProfile +} from "../../api/mail"; + +const DOCUMENTATION = { + topicId: "mail.workflow.legacy-pop3-import", + documentationType: "admin" +} as const; + +type Source = { + profile: MailServerProfile; + server: MailPop3ServerEndpoint; +}; + +type SourceDraft = { + profileId: string; + name: string; + host: string; + port: string; + security: "tls" | "starttls" | "plain"; + timeoutSeconds: string; + maxMessageMiB: string; + maxBatchMiB: string; + previewBodyLines: string; + username: string; + password: string; + enabled: boolean; + allowDeleteAfterImport: boolean; +}; + +const EMPTY_DRAFT: SourceDraft = { + profileId: "", + name: "Legacy POP3 source", + host: "", + port: "995", + security: "tls", + timeoutSeconds: "30", + maxMessageMiB: "25", + maxBatchMiB: "100", + previewBodyLines: "20", + username: "", + password: "", + enabled: true, + allowDeleteAfterImport: false +}; + +export default function MailLegacyImportPage({ + settings, + auth +}: { + settings: ApiSettings; + auth: AuthInfo; +}) { + const navigate = useGuardedNavigate(); + const [profiles, setProfiles] = useState([]); + const [imports, setImports] = useState([]); + const [selectedSourceId, setSelectedSourceId] = useState(""); + const [preview, setPreview] = useState(null); + const [selectedUidls, setSelectedUidls] = useState([]); + const [deleteAfterImport, setDeleteAfterImport] = useState(false); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState(""); + const [error, setError] = useState(""); + const [success, setSuccess] = useState(""); + const [sourceDialogOpen, setSourceDialogOpen] = useState(false); + const [editingSourceId, setEditingSourceId] = useState(null); + const [sourceDraft, setSourceDraft] = useState(EMPTY_DRAFT); + const [deleteConfirmationOpen, setDeleteConfirmationOpen] = useState(false); + + const canImport = hasScope(auth, "mail:pop3:import"); + const canDelete = hasScope(auth, "mail:pop3:delete"); + const canManage = hasScope(auth, "mail:pop3:manage"); + const canManageSecrets = hasScope(auth, "mail:secret:manage"); + const sources = useMemo(() => pop3Sources(profiles), [profiles]); + const selectedSource = sources.find((item) => item.server.id === selectedSourceId) ?? sources[0] ?? null; + const selectedCredential = selectedSource ? defaultCredential(selectedSource.server) : null; + const configurableProfiles = profiles.filter((profile) => profileCanBeConfigured(auth, profile)); + const sourceCanBeConfigured = selectedSource ? profileCanBeConfigured(auth, selectedSource.profile) : false; + + async function load() { + setLoading(true); + setError(""); + try { + const [nextProfiles, nextImports] = await Promise.all([ + listMailServerProfiles(settings, canManage), + canImport ? listMailPop3Imports(settings, null, 200) : Promise.resolve([]) + ]); + const nextSources = pop3Sources(nextProfiles); + setProfiles(nextProfiles); + setImports(nextImports); + setSelectedSourceId((current) => + nextSources.some((item) => item.server.id === current) + ? current + : nextSources[0]?.server.id ?? "" + ); + } catch (reason) { + setError(adminErrorMessage(reason)); + } finally { + setLoading(false); + } + } + + useEffect(() => { + void load(); + }, [settings.apiBaseUrl, settings.apiKey, settings.accessToken, canImport]); + + useEffect(() => { + setPreview(null); + setSelectedUidls([]); + setDeleteAfterImport(false); + }, [selectedSourceId]); + + async function runConnectionTest() { + if (!selectedSource) return; + setBusy("test"); + setError(""); + setSuccess(""); + try { + const result = await testMailProfilePop3( + settings, + selectedSource.profile.id, + selectedSource.server.id, + selectedCredential?.id + ); + if (!result.ok) throw new Error(result.message); + setSuccess( + `POP3 authentication succeeded. The mailbox currently reports ${String(result.details.message_count ?? 0)} message(s).` + ); + } catch (reason) { + setError(adminErrorMessage(reason)); + } finally { + setBusy(""); + } + } + + async function refreshPreview() { + if (!selectedSource || !canImport) return; + setBusy("preview"); + setError(""); + setSuccess(""); + try { + const next = await previewMailProfilePop3(settings, selectedSource.profile.id, { + server_id: selectedSource.server.id, + credential_id: selectedCredential?.id, + limit: 100 + }); + setPreview(next); + setSelectedUidls((current) => + current.filter((uidl) => next.messages.some((item) => item.uidl === uidl && !item.already_imported)) + ); + } catch (reason) { + setError(adminErrorMessage(reason)); + } finally { + setBusy(""); + } + } + + async function runImport() { + if (!selectedSource || !preview || selectedUidls.length === 0) return; + setDeleteConfirmationOpen(false); + setBusy("import"); + setError(""); + setSuccess(""); + try { + const result = await importMailProfilePop3(settings, selectedSource.profile.id, { + server_id: selectedSource.server.id, + credential_id: selectedCredential?.id, + expected_transport_revision: preview.transport_revision, + uidls: selectedUidls, + delete_after_import: deleteAfterImport + }); + setSuccess( + `${result.imports.length} message(s) imported; ${result.duplicate_uidls.length} duplicate(s) skipped. Source deletion: ${result.deletion_status.replaceAll("_", " ")}.` + ); + setSelectedUidls([]); + const [nextPreview, nextImports] = await Promise.all([ + previewMailProfilePop3(settings, selectedSource.profile.id, { + server_id: selectedSource.server.id, + credential_id: selectedCredential?.id, + limit: 100 + }), + listMailPop3Imports(settings, null, 200) + ]); + setPreview(nextPreview); + setImports(nextImports); + } catch (reason) { + setError(adminErrorMessage(reason)); + } finally { + setBusy(""); + } + } + + function openSourceDialog(source: Source | null) { + const credential = source ? defaultCredential(source.server) : null; + setEditingSourceId(source?.server.id ?? null); + setSourceDraft(source ? sourceDraftFromSource(source, credential) : { + ...EMPTY_DRAFT, + profileId: selectedSource?.profile.id ?? configurableProfiles[0]?.id ?? "" + }); + setSourceDialogOpen(true); + } + + async function saveSource() { + const profile = profiles.find((item) => item.id === sourceDraft.profileId); + if (!profile) return; + const existing = sources.find((item) => item.server.id === editingSourceId) ?? null; + const existingCredential = existing ? defaultCredential(existing.server) : null; + setBusy("source"); + setError(""); + setSuccess(""); + try { + if (existing) { + if (canManageSecrets && (sourceDraft.password || !existingCredential)) { + if (existingCredential) { + await updateMailServerCredential( + settings, + existing.profile.id, + existing.server.id, + existingCredential.id, + { + name: `${sourceDraft.name.trim()} credential`, + username: sourceDraft.username.trim(), + ...(sourceDraft.password ? { password: sourceDraft.password } : {}) + } + ); + } else { + await createMailServerCredential( + settings, + existing.profile.id, + existing.server.id, + credentialPayload(sourceDraft, existing.server.id) + ); + } + } + await updateMailServerEndpoint(settings, existing.profile.id, existing.server.id, { + name: sourceDraft.name.trim(), + config: sourceConfig(sourceDraft), + is_active: true + }); + setSuccess("Legacy POP3 source updated."); + } else { + const disabledServer = await createMailServerEndpoint(settings, profile.id, { + protocol: "pop3", + name: sourceDraft.name.trim(), + config: { + ...sourceConfig(sourceDraft), + legacy_import_enabled: false, + allow_delete_after_import: false + }, + is_default: false, + is_active: false + }); + await createMailServerCredential( + settings, + profile.id, + disabledServer.id, + credentialPayload(sourceDraft, disabledServer.id) + ); + await updateMailServerEndpoint(settings, profile.id, disabledServer.id, { + config: sourceConfig(sourceDraft), + is_active: true + }); + setSelectedSourceId(disabledServer.id); + setSuccess("Legacy POP3 source created. It was enabled only after its encrypted credential was stored."); + } + setSourceDialogOpen(false); + await load(); + } catch (reason) { + setError(adminErrorMessage(reason)); + } finally { + setBusy(""); + } + } + + const sourceBlocker = sourceSaveBlocker({ + draft: sourceDraft, + existingCredential: editingSourceId ? defaultCredential(sources.find((item) => item.server.id === editingSourceId)?.server) : null, + canManageSecrets + }); + const importBlocker = !selectedSource + ? "Select an enabled legacy POP3 source." + : !preview + ? "Refresh the live preview before importing." + : selectedUidls.length === 0 + ? "Select at least one message that has not already been imported." + : deleteAfterImport && (!canDelete || !preview.delete_after_import_allowed) + ? "Source deletion needs both the destructive permission and an endpoint policy that allows it." + : ""; + + const previewColumns: DataGridColumn[] = [ + { + id: "select", + header: "Select", + width: 82, + render: (item) => setSelectedUidls((current) => + current.includes(item.uidl) + ? current.filter((uidl) => uidl !== item.uidl) + : [...current, item.uidl] + )} /> + }, + { + id: "subject", + header: "Message", + width: "minmax(240px, 1.3fr)", + filterable: true, + value: (item) => `${item.subject || ""} ${item.from_header || ""}`, + render: (item) => {item.subject || "No subject"}
{item.from_header || "Unknown sender"} + }, + { id: "date", header: "Provider date", width: "minmax(170px, .8fr)", value: (item) => item.date || "", render: (item) => item.date || "Unknown" }, + { id: "size", header: "Size", width: 105, value: (item) => item.size_bytes, render: (item) => formatBytes(item.size_bytes) }, + { + id: "state", + header: "State", + width: 130, + value: (item) => item.already_imported ? "imported" : "available", + render: (item) => + } + ]; + + const importColumns: DataGridColumn[] = [ + { id: "imported", header: "Imported", width: "minmax(170px, .8fr)", value: (item) => item.imported_at, render: (item) => formatDateTime(item.imported_at) }, + { id: "subject", header: "Message", width: "minmax(240px, 1.2fr)", filterable: true, value: (item) => `${item.subject || ""} ${item.from_header || ""}`, render: (item) => {item.subject || "No subject"}
{item.from_header || "Unknown sender"} }, + { id: "review", header: "Review state", width: 140, value: (item) => item.status, render: (item) => }, + { id: "deletion", header: "Source deletion", width: 165, value: (item) => item.deletion_status, render: (item) => } + ]; + + return ( + <> + void load(), disabled: Boolean(busy), disabledReason: busy ? "Wait for the current POP3 action to finish." : undefined }} + contextActions={} + helpAction={} + createAction={canManage ? : undefined} + />} + loading={loading} + loadingLabel="Loading legacy POP3 sources" + error={error} + success={success} + documentationType="admin" + > + + + {sources.length === 0 ? : + {sources.map((source) => setSelectedSourceId(source.server.id)}> + } + /> + )} + } + + + + + + + {preview ? <> +

Provider reports {preview.message_count} message(s), {formatBytes(preview.mailbox_size_bytes)} total. Preview and ordinary import do not delete source messages.

+ item.uidl} emptyText="The legacy mailbox contains no messages." /> + + + + + :

Refresh a source to obtain a bounded, non-destructive preview.

} + + + + item.id} emptyText="No legacy messages have been imported." /> + + + + !busy && setSourceDialogOpen(false)} + footer={<>} + > + + + + + setSourceDraft((draft) => ({ ...draft, name: event.target.value }))} /> + setSourceDraft((draft) => ({ ...draft, host: event.target.value }))} placeholder="pop3.example.org" /> + setSourceDraft((draft) => ({ ...draft, port: event.target.value }))} /> + + + + setSourceDraft((draft) => ({ ...draft, timeoutSeconds: event.target.value }))} /> + setSourceDraft((draft) => ({ ...draft, maxMessageMiB: event.target.value }))} /> + setSourceDraft((draft) => ({ ...draft, maxBatchMiB: event.target.value }))} /> + setSourceDraft((draft) => ({ ...draft, previewBodyLines: event.target.value }))} /> + setSourceDraft((draft) => ({ ...draft, username: event.target.value }))} autoComplete="username" /> + setSourceDraft((draft) => ({ ...draft, password: event.target.value }))} autoComplete="new-password" /> + setSourceDraft((draft) => ({ ...draft, enabled, allowDeleteAfterImport: enabled ? draft.allowDeleteAfterImport : false }))} label="Explicitly enable legacy import" help="Off is the product default." /> + setSourceDraft((draft) => ({ ...draft, allowDeleteAfterImport }))} label="Permit delete-after-import requests" help="Operators still need a separate destructive permission and must choose deletion per batch." /> + + + + setDeleteConfirmationOpen(false)} + onConfirm={() => void runImport()} + /> + + ); +} + +function pop3Sources(profiles: MailServerProfile[]): Source[] { + return profiles.flatMap((profile) => + ((profile.servers ?? []) as unknown as MailPop3ServerEndpoint[]) + .filter((server) => server.protocol === "pop3") + .map((server) => ({ profile, server })) + ); +} + +function defaultCredential(server: MailPop3ServerEndpoint | undefined): MailCredentialEnvelope | null { + if (!server) return null; + return server.credentials.find((credential) => credential.is_default) + ?? server.credentials.find((credential) => credential.is_active) + ?? server.credentials[0] + ?? null; +} + +function profileCanBeConfigured(auth: AuthInfo, profile: MailServerProfile): boolean { + if (profile.scope_type === "system") return hasScope(auth, "system:settings:write"); + if (hasScope(auth, "mail:profile:write")) return true; + return profile.scope_type === "user" + && profile.scope_id === auth.user.id + && hasScope(auth, "mail:profile:write_own"); +} + +function sourceDraftFromSource(source: Source, credential: MailCredentialEnvelope | null): SourceDraft { + const config = source.server.config; + return { + profileId: source.profile.id, + name: source.server.name, + host: String(config.host ?? ""), + port: String(config.port ?? (config.security === "tls" ? 995 : 110)), + security: config.security === "starttls" || config.security === "plain" ? config.security : "tls", + timeoutSeconds: String(config.timeout_seconds ?? 30), + maxMessageMiB: String(Math.max(1, Math.round(Number(config.max_message_bytes ?? 25 * 1024 * 1024) / 1024 / 1024))), + maxBatchMiB: String(Math.max(1, Math.round(Number(config.max_batch_bytes ?? 100 * 1024 * 1024) / 1024 / 1024))), + previewBodyLines: String(config.preview_body_lines ?? 20), + username: String(credential?.public_data?.username ?? ""), + password: "", + enabled: Boolean(config.legacy_import_enabled), + allowDeleteAfterImport: Boolean(config.allow_delete_after_import) + }; +} + +function sourceConfig(draft: SourceDraft): Record { + return { + host: draft.host.trim(), + port: Number(draft.port), + security: draft.security, + timeout_seconds: Number(draft.timeoutSeconds), + max_message_bytes: Number(draft.maxMessageMiB) * 1024 * 1024, + max_batch_bytes: Number(draft.maxBatchMiB) * 1024 * 1024, + preview_body_lines: Number(draft.previewBodyLines), + legacy_import_enabled: draft.enabled, + allow_delete_after_import: draft.allowDeleteAfterImport + }; +} + +function credentialPayload(draft: SourceDraft, serverId: string) { + return { + name: `${draft.name.trim()} credential`, + credential_kind: "username_password", + username: draft.username.trim(), + password: draft.password, + allowed_modules: ["mail"], + allowed_server_refs: [`mail:${serverId}`], + is_default: true + }; +} + +function sourceSaveBlocker({ + draft, + existingCredential, + canManageSecrets +}: { + draft: SourceDraft; + existingCredential: MailCredentialEnvelope | null; + canManageSecrets: boolean; +}): string { + if (!draft.profileId) return "Select a Mail profile."; + if (!draft.name.trim()) return "Enter a source name."; + if (!draft.host.trim()) return "Enter the POP3 host."; + if (!boundedInteger(draft.port, 1, 65535)) return "Enter a valid POP3 port."; + if (!boundedInteger(draft.timeoutSeconds, 1, 300)) return "Enter a timeout from 1 to 300 seconds."; + if (!boundedInteger(draft.maxMessageMiB, 1, 50)) return "Enter a message limit from 1 to 50 MiB."; + if (!boundedInteger(draft.maxBatchMiB, 1, 500)) return "Enter a batch limit from 1 to 500 MiB."; + if (Number(draft.maxBatchMiB) < Number(draft.maxMessageMiB)) return "The batch limit cannot be lower than the per-message limit."; + if (!boundedInteger(draft.previewBodyLines, 0, 100)) return "Enter 0 to 100 preview body lines."; + if (draft.allowDeleteAfterImport && !draft.enabled) return "Enable legacy import before permitting source deletion."; + if (!existingCredential && !canManageSecrets) return "Managing the encrypted POP3 credential requires Mail secret authority."; + if (!existingCredential && !draft.username.trim()) return "Enter the POP3 username."; + if (!existingCredential && !draft.password) return "Enter the POP3 password."; + return ""; +} + +function boundedInteger(value: string, minimum: number, maximum: number): boolean { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= minimum && parsed <= maximum; +} + +function deletionTone(status: string): "success" | "warning" | "error" | "inactive" { + if (status === "succeeded") return "success"; + if (status === "failed" || status === "outcome_unknown") return "error"; + if (status === "pending") return "warning"; + return "inactive"; +} + +function formatBytes(value: number): string { + if (value < 1024) return `${value} B`; + if (value < 1024 * 1024) return `${(value / 1024).toFixed(1)} KiB`; + return `${(value / 1024 / 1024).toFixed(1)} MiB`; +} diff --git a/webui/src/module.ts b/webui/src/module.ts index eae561e..d236689 100644 --- a/webui/src/module.ts +++ b/webui/src/module.ts @@ -9,8 +9,10 @@ import "./styles/mail-profiles.css"; const MailboxPage = lazy(() => import("./features/mail/MailboxPage")); const MailBouncePage = lazy(() => import("./features/mail/MailBouncePage")); +const MailLegacyImportPage = lazy(() => import("./features/mail/MailLegacyImportPage")); const mailboxRead = ["mail:mailbox:read"]; const bounceRead = ["mail:bounce:read", "mail:bounce:manage"]; +const legacyImportAccess = ["mail:pop3:import", "mail:pop3:manage"]; const translations = { en: generatedTranslations.en, de: generatedTranslations.de @@ -39,10 +41,14 @@ export const mailModule: PlatformWebModule = { { id: "mail.settings.profiles", moduleId: "mail", kind: "section", label: "Personal mail profiles", order: 10 }, { id: "mail.quick_access.messages", moduleId: "mail", kind: "quick_access", label: "Mail Quick Access", order: 80 } ], - navItems: [{ to: "/mail", label: "i18n:govoplan-mail.mail.92379cbb", iconName: "mail", anyOf: mailboxRead, order: 50 }], + navItems: [ + { to: "/mail", label: "i18n:govoplan-mail.mail.92379cbb", iconName: "mail", anyOf: mailboxRead, order: 50 }, + { to: "/mail/legacy-import", label: "Legacy POP3 import", iconName: "mail", anyOf: legacyImportAccess, order: 52 } + ], routes: [ { path: "/mail", anyOf: mailboxRead, order: 50, render: ({ settings, auth }) => createElement(MailboxPage, { settings, auth }) }, - { path: "/mail/bounces", anyOf: bounceRead, order: 51, render: ({ settings }) => createElement(MailBouncePage, { settings }) }], + { path: "/mail/bounces", anyOf: bounceRead, order: 51, render: ({ settings }) => createElement(MailBouncePage, { settings }) }, + { path: "/mail/legacy-import", anyOf: legacyImportAccess, order: 52, render: ({ settings, auth }) => createElement(MailLegacyImportPage, { settings, auth }) }], uiCapabilities: { "mail.profiles": { MailProfileScopeManager, MailProfilePolicyEditor, validateMailPolicy } satisfies MailProfilesUiCapability,