GovOPlaN/govoplan-campaign
1
0
Fork 0
Code Issues 46 Pull Requests Actions Packages Projects Releases Wiki Activity

security: harden Campaign delivery effects and reconciliation

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-21 17:14:33 +02:00
parent 057e660b17
commit 09c63de813
25 changed files with 2079 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/govoplan_campaign/backend/campaign/loader.py
View File

@@ -7,6 +7,7 @@ from typing import Any
from jsonschema import Draft202012Validator, FormatChecker
from .mail_profile_boundary import assert_campaign_uses_mail_profile_reference
from .models import CampaignConfig
@@ -83,6 +84,7 @@ def load_campaign_config(
schema_path: str | Path | None = None,
) -> CampaignConfig:
data = load_campaign_json(path)
assert_campaign_uses_mail_profile_reference(data)
if validate_schema:
validate_against_schema(data, schema_path=schema_path)
return CampaignConfig.model_validate(data)

40
src/govoplan_campaign/backend/campaign/models.py
View File

@@ -6,15 +6,6 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from govoplan_core.mail.config import (
ImapConfig,
ImapServerConfig,
SmtpConfig,
SmtpServerConfig,
TransportCredentials,
normalize_split_transport_credentials,
)
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", populate_by_name=True)
@@ -115,37 +106,14 @@ class FieldDefinition(StrictModel):
can_override: bool = True
class MailServerCredentials(StrictModel):
smtp: TransportCredentials = Field(default_factory=TransportCredentials)
imap: TransportCredentials = Field(default_factory=TransportCredentials)
class MailProfileCapabilities(StrictModel):
smtp_available: bool = False
imap_available: bool = False
class ServerConfig(StrictModel):
mail_profile_id: str | None = None
inherit_smtp_credentials: bool = True
inherit_imap_credentials: bool = True
smtp: SmtpServerConfig | None = None
imap: ImapServerConfig | None = None
credentials: MailServerCredentials = Field(default_factory=MailServerCredentials)
@model_validator(mode="before")
@classmethod
def normalize_legacy_credentials(cls, value: Any) -> Any:
return normalize_split_transport_credentials(value)
def runtime_smtp_config(self) -> SmtpConfig | None:
if self.smtp is None:
return None
payload = self.smtp.model_dump(mode="json")
payload.update(self.credentials.smtp.model_dump(mode="json", exclude_none=True))
return SmtpConfig.model_validate(payload)
def runtime_imap_config(self) -> ImapConfig | None:
if self.imap is None:
return None
payload = self.imap.model_dump(mode="json")
payload.update(self.credentials.imap.model_dump(mode="json", exclude_none=True))
return ImapConfig.model_validate(payload)
profile_capabilities: MailProfileCapabilities = Field(default_factory=MailProfileCapabilities)
class RecipientConfig(StrictModel):

74
src/govoplan_campaign/backend/campaign/validation.py
View File

@@ -8,6 +8,7 @@ from typing import Iterable
from pydantic import BaseModel, ConfigDict, Field
from .addressing import effective_address_lists
from .field_values import ignored_entry_field_overrides
from .models import AttachmentConfig, CampaignConfig, EntryConfig, FieldType, SourceType, ZipArchiveConfig, ZipPasswordMode, ZipPasswordScope, ZipRuleMode
from ..attachments.resolver import resolve_campaign_attachments
@@ -338,52 +339,72 @@ def _global_value_issues(config: CampaignConfig, declared_names: set[str]) -> li
def _delivery_issues(config: CampaignConfig) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
runtime_imap = config.server.runtime_imap_config()
if config.delivery.imap_append_sent.enabled:
issues.extend(_imap_delivery_issues(runtime_imap))
runtime_smtp = config.server.runtime_smtp_config()
if config.campaign.mode == "send" and not runtime_smtp:
profile_id = (config.server.mail_profile_id or "").strip()
if (config.campaign.mode == "send" or config.delivery.imap_append_sent.enabled) and not profile_id:
issues.append(
_issue(
Severity.ERROR,
"missing_smtp_config",
"campaign mode is 'send', but no server.smtp configuration is present",
"/server/smtp",
"missing_mail_profile",
"Select an authorized Mail-module profile; campaigns cannot store SMTP/IMAP settings or credentials.",
"/server/mail_profile_id",
)
)
if runtime_smtp:
missing = [name for name in ["host", "port"] if getattr(runtime_smtp, name) in (None, "")]
if missing:
capabilities = config.server.profile_capabilities
if config.campaign.mode == "send" and profile_id and not capabilities.smtp_available:
issues.append(
_issue(
Severity.WARNING,
"incomplete_smtp_config",
"SMTP settings are present, but these settings are missing: " + ", ".join(missing),
"/server/smtp",
Severity.ERROR,
"mail_profile_without_smtp",
"campaign mode is 'send', but the selected Mail profile has no SMTP configuration",
"/server/mail_profile_id",
)
)
if config.delivery.imap_append_sent.enabled and profile_id and not capabilities.imap_available:
issues.append(
_issue(
Severity.ERROR,
"mail_profile_without_imap",
"IMAP append is enabled, but the selected Mail profile has no IMAP configuration",
"/server/mail_profile_id",
)
)
return issues
def _imap_delivery_issues(runtime_imap: object | None) -> list[SemanticIssue]:
if runtime_imap is None:
def _sender_issues(config: CampaignConfig) -> list[SemanticIssue]:
"""Require Campaign-owned sender data before a send-mode build."""
if config.campaign.mode != "send":
return []
if config.entries.is_inline:
return [
_issue(
Severity.WARNING,
"delivery_imap_enabled_without_server_imap",
"delivery.imap_append_sent is enabled, but no server.imap configuration is present",
"/delivery/imap_append_sent/enabled",
Severity.ERROR,
"missing_sender",
"No effective From address is configured; Campaign must resolve the sender before building.",
f"/entries/inline/{index}/from",
)
for index, entry in enumerate(config.entries.inline or [])
if entry.active and not effective_address_lists(config, entry)["from"]
]
missing = [name for name in ["host", "port", "username", "password"] if getattr(runtime_imap, name) in (None, "")]
if not missing:
if config.recipients.from_:
return []
defaults = config.entries.defaults
mapping_can_supply_sender = bool(
config.recipients.allow_individual_from
and (
(defaults is not None and defaults.from_)
or "from.email" in (config.entries.mapping or {})
)
)
if mapping_can_supply_sender:
return []
return [
_issue(
Severity.ERROR,
"incomplete_imap_config",
"IMAP append is enabled, but these IMAP settings are missing: " + ", ".join(missing),
"/server/imap",
"missing_sender",
"Configure recipients.from, or allow and map an individual From address, before building a send-mode campaign.",
"/recipients/from",
)
]
@@ -602,6 +623,7 @@ def validate_campaign_config(
issues.extend(_attachment_path_issues(config))
issues.extend(_zip_configuration_issues(config))
issues.extend(_delivery_issues(config))
issues.extend(_sender_issues(config))
entries = _entries_validation(
config,

72
src/govoplan_campaign/backend/integrations.py
View File

@@ -34,9 +34,16 @@ class ImapConfigurationError(RuntimeError):
class ImapAppendError(RuntimeError):
def __init__(self, message: str, *, temporary: bool | None = None) -> None:
def __init__(
self,
message: str,
*,
temporary: bool | None = None,
outcome_unknown: bool = False,
) -> None:
super().__init__(message)
self.temporary = temporary
self.outcome_unknown = outcome_unknown
class MailProfileError(OptionalModuleUnavailable):
@@ -139,17 +146,6 @@ class MailCampaignIntegration:
raise MailProfileError("Mail module is not available")
return self._delegate
def materialize_campaign_mail_profile_config(self, session: Any, **kwargs: Any) -> dict[str, Any]:
if self._delegate is None:
raw_json = kwargs.get("raw_json")
if self.mail_profile_id_from_campaign_json(raw_json if isinstance(raw_json, dict) else {}):
raise MailProfileError("Campaign mail-server profiles require the mail module")
return dict(raw_json) if isinstance(raw_json, dict) else {}
try:
return self._delegate.materialize_campaign_mail_profile_config(session, **kwargs)
except getattr(self._delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def assert_campaign_mail_policy_allows_json(self, session: Any, **kwargs: Any) -> None:
if self._delegate is None:
raw_json = kwargs.get("raw_json")
@@ -162,72 +158,50 @@ class MailCampaignIntegration:
except getattr(self._delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def assert_mail_policy_allows_send(self, session: Any, **kwargs: Any) -> None:
delegate = self._require()
try:
return delegate.assert_mail_policy_allows_send(session, **kwargs)
except getattr(delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def mail_profile_id_from_campaign_json(self, raw_json: dict[str, Any]) -> str | None:
if self._delegate is not None:
return self._delegate.mail_profile_id_from_campaign_json(raw_json)
server = raw_json.get("server") if isinstance(raw_json, dict) else None
profile_id = server.get("mail_profile_id") if isinstance(server, dict) else None
if profile_id is None and isinstance(server, dict):
profile_id = server.get("profile_id")
return str(profile_id).strip() if profile_id else None
def ensure_mail_profile_allowed_for_campaign(self, session: Any, **kwargs: Any) -> Any:
def campaign_profile_delivery_summary(self, session: Any, **kwargs: Any) -> dict[str, Any]:
delegate = self._require()
try:
return delegate.ensure_mail_profile_allowed_for_campaign(session, **kwargs)
return delegate.campaign_profile_delivery_summary(session, **kwargs)
except getattr(delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def smtp_config_from_profile(self, profile: Any) -> Any:
return self._require().smtp_config_from_profile(profile)
def imap_config_from_profile(self, profile: Any) -> Any:
return self._require().imap_config_from_profile(profile)
def effective_profile_credentials_inherited(self, session: Any, **kwargs: Any) -> bool:
return self._require().effective_profile_credentials_inherited(session, **kwargs)
def apply_campaign_credentials(self, profile_payload: dict[str, Any], server: dict[str, Any], protocol: str) -> dict[str, Any]:
return self._require().apply_campaign_credentials(profile_payload, server, protocol)
def wait_for_rate_limit(self, **kwargs: Any) -> None:
if self._delegate is None:
return None
return self._delegate.wait_for_rate_limit(**kwargs)
def send_email_bytes(self, *args: Any, **kwargs: Any) -> Any:
def send_campaign_email_bytes(self, *args: Any, **kwargs: Any) -> Any:
delegate = self._require()
try:
return delegate.send_email_bytes(*args, **kwargs)
return delegate.send_campaign_email_bytes(*args, **kwargs)
except getattr(delegate, "SmtpSendError", SmtpSendError) as exc:
raise SmtpSendError(str(exc), temporary=bool(getattr(exc, "temporary", False)), outcome_unknown=bool(getattr(exc, "outcome_unknown", False))) from exc
except getattr(delegate, "SmtpConfigurationError", SmtpConfigurationError) as exc:
raise SmtpConfigurationError(str(exc)) from exc
except getattr(delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def append_message_to_sent(self, *args: Any, **kwargs: Any) -> Any:
def append_campaign_message_to_sent(self, *args: Any, **kwargs: Any) -> Any:
delegate = self._require()
try:
return delegate.append_message_to_sent(*args, **kwargs)
return delegate.append_campaign_message_to_sent(*args, **kwargs)
except getattr(delegate, "ImapAppendError", ImapAppendError) as exc:
raise ImapAppendError(str(exc), temporary=getattr(exc, "temporary", None)) from exc
raise ImapAppendError(
str(exc),
temporary=getattr(exc, "temporary", None),
outcome_unknown=bool(getattr(exc, "outcome_unknown", False)),
) from exc
except getattr(delegate, "ImapConfigurationError", ImapConfigurationError) as exc:
raise ImapConfigurationError(str(exc)) from exc
def send_email_message(self, *args: Any, **kwargs: Any) -> Any:
delegate = self._require()
try:
return delegate.send_email_message(*args, **kwargs)
except getattr(delegate, "SmtpSendError", SmtpSendError) as exc:
raise SmtpSendError(str(exc), temporary=bool(getattr(exc, "temporary", False)), outcome_unknown=bool(getattr(exc, "outcome_unknown", False))) from exc
except getattr(delegate, "SmtpConfigurationError", SmtpConfigurationError) as exc:
raise SmtpConfigurationError(str(exc)) from exc
except getattr(delegate, "MailProfileError", MailProfileError) as exc:
raise MailProfileError(str(exc)) from exc
def mock_mailbox(self) -> Any | None:
if self._delegate is None or not hasattr(self._delegate, "mock_mailbox"):

123
src/govoplan_campaign/backend/manifest.py
View File

@@ -12,6 +12,9 @@ from govoplan_core.core.campaigns import (
)
from govoplan_core.core.module_guards import drop_table_retirement_provider, persistent_table_uninstall_guard
from govoplan_core.core.modules import (
DocumentationCondition,
DocumentationLink,
DocumentationTopic,
FrontendModule,
MigrationSpec,
ModuleContext,
@@ -59,7 +62,7 @@ PERMISSIONS = (
_permission("campaigns:campaign:control", "Control delivery", "Pause, resume or cancel queued and sending jobs.", "Campaigns"),
_permission("campaigns:campaign:send", "Send campaigns", "Start real SMTP delivery.", "Campaigns"),
_permission("campaigns:campaign:retry", "Retry delivery", "Retry failed or unattempted delivery jobs.", "Campaigns"),
_permission("campaigns:campaign:reconcile", "Reconcile delivery", "Resolve outcome-unknown SMTP attempts after inspection.", "Campaigns"),
_permission("campaigns:campaign:reconcile", "Reconcile delivery", "Resolve outcome-unknown SMTP or IMAP attempts after inspection.", "Campaigns"),
_permission("campaigns:diagnostic:read", "View campaign diagnostics", "Inspect worker claims and internal storage locators for campaign delivery troubleshooting.", "Campaign operations"),
_permission("campaigns:recipient:read", "View recipients", "Read recipient lists and recipient-specific campaign data.", "Recipients"),
_permission("campaigns:recipient:write", "Edit recipients", "Create and edit recipient rows and field values.", "Recipients"),
@@ -144,7 +147,7 @@ def _campaigns_router(context: ModuleContext):
manifest = ModuleManifest(
id="campaigns",
name="Campaigns",
version="0.1.8",
version="0.1.9",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("files", "mail", "notifications", "addresses"),
provides_interfaces=(
@@ -163,8 +166,8 @@ manifest = ModuleManifest(
),
ModuleInterfaceRequirement(
name="mail.campaign_delivery",
version_min="0.1.0",
version_max_exclusive="0.2.0",
version_min="0.2.0",
version_max_exclusive="0.3.0",
optional=True,
),
ModuleInterfaceRequirement(
@@ -258,6 +261,118 @@ manifest = ModuleManifest(
label="Campaigns",
),
),
documentation=(
DocumentationTopic(
id="campaigns.mail-profile-user-journey",
title="Choose a Mail profile for campaign delivery",
summary="Campaigns reference an authorized Mail profile and never store SMTP/IMAP settings or credentials.",
body="Open the campaign Mail settings, select an available profile, test it through Mail, and save. Validation and delivery recheck profile authorization. A changed transport identity requires a new validation and build.",
layer="available",
documentation_types=("user",),
audience=("campaign_manager", "campaign_reviewer", "campaign_sender"),
order=46,
conditions=(
DocumentationCondition(
required_modules=("campaigns", "mail"),
any_scopes=("campaigns:campaign:read", "mail:profile:use"),
),
),
links=(
DocumentationLink(label="Campaigns", href="/campaigns", kind="runtime"),
DocumentationLink(label="Mail profiles", href="/mail", kind="runtime"),
),
related_modules=("mail",),
unlocks=("Profile-backed SMTP delivery and optional IMAP append-to-Sent.",),
metadata={
"kind": "workflow",
"route": "/campaigns/{campaign_id}/mail",
"screen": "Campaign Mail settings",
"prerequisites": [
"Campaign and Mail are installed.",
"You may edit the current campaign version and use at least one authorized Mail profile.",
],
"steps": [
"Open the campaign and go to Mail settings.",
"Select an available Mail profile; Campaign stores only its stable identifier.",
"Test SMTP and, when configured, IMAP through the Mail module.",
"Save, validate, and build the campaign before queueing delivery.",
],
"outcome": "The editable campaign version references an authorized Mail-owned delivery profile without copying transport settings or credentials.",
"verification": "Reopen Mail settings, confirm the selected profile, then run validation and verify that the build completes without profile-drift errors.",
"related_topic_ids": [
"campaigns.mail-profile-governance",
"campaigns.mail-profile-operations",
"mail.profile-ownership-and-consumers",
],
},
),
DocumentationTopic(
id="campaigns.mail-profile-governance",
title="Govern Campaign-to-Mail profile references",
summary="Mail owns transport definitions and encrypted credentials; Campaign owns only the selected profile reference and delivery evidence.",
body="Grant mail:profile:use to campaign authors, constrain profile availability through Mail policy, and keep effective credential inheritance enabled. Inline transport fields are rejected. Legacy records remain unchanged until an explicit, audited profile migration creates or updates an editable version.",
layer="configured",
documentation_types=("admin",),
audience=("tenant_admin", "mail_admin", "campaign_admin"),
order=47,
conditions=(
DocumentationCondition(
required_modules=("campaigns", "mail"),
any_scopes=("mail:profile:write", "admin:policies:read", "system:settings:read"),
),
),
links=(
DocumentationLink(label="Mail profiles", href="/mail", kind="runtime"),
DocumentationLink(label="Campaign schema", href="/api/v1/campaigns/schema", kind="api"),
),
related_modules=("mail", "access"),
unlocks=("Auditable, reusable transport configuration across campaigns.",),
metadata={
"kind": "reference",
"route": "/campaigns/{campaign_id}/mail-policy",
"screen": "Campaign Mail policy",
"section": "Profile authorization and credential inheritance",
"related_topic_ids": [
"campaigns.mail-profile-user-journey",
"campaigns.mail-profile-operations",
"mail.profile-ownership-and-consumers",
],
},
),
DocumentationTopic(
id="campaigns.mail-profile-operations",
title="Operate profile-backed campaign delivery",
summary="Workers re-authorize and resolve Mail profiles at execution time while Campaign retains only opaque Mail-owned revisions and outcomes.",
body="A legacy snapshot, unauthorized or inactive profile, profile-reference mismatch, or changed SMTP/IMAP transport revision stops delivery. Preserve the record, migrate or correct the profile selection, revalidate, rebuild, and only then queue again. Password-only rotation remains possible without copying secrets into Campaign. Uncertain SMTP and IMAP effects remain blocked until an evidence-backed operator reconciliation.",
layer="configured",
documentation_types=("admin",),
audience=("campaign_sender", "campaign_operator", "mail_admin"),
order=48,
conditions=(
DocumentationCondition(
required_modules=("campaigns", "mail"),
any_scopes=("campaigns:diagnostic:read", "campaigns:campaign:reconcile", "mail:profile:test"),
),
),
links=(
DocumentationLink(label="Campaign operator queue", href="/operator", kind="runtime"),
DocumentationLink(label="Campaign reports", href="/reports", kind="runtime"),
),
related_modules=("mail", "audit"),
unlocks=("Fail-closed recovery without exposing Mail credentials.",),
metadata={
"kind": "reference",
"route": "/operator",
"screen": "Campaign operator queue",
"section": "Profile-backed delivery recovery",
"related_topic_ids": [
"campaigns.mail-profile-user-journey",
"campaigns.mail-profile-governance",
"mail.profile-ownership-and-consumers",
],
},
),
),
capability_factories={
CAPABILITY_CAMPAIGNS_ACCESS: lambda context: __import__(
"govoplan_campaign.backend.capabilities",

24
src/govoplan_campaign/backend/messages/builder.py
View File

@@ -590,6 +590,25 @@ def _validate_required_recipients(
return MessageValidationStatus.EXCLUDED
def _validate_required_sender(
senders: list[RecipientConfig],
issues: list[MessageIssue],
validation_status: MessageValidationStatus,
) -> MessageValidationStatus:
if senders:
return validation_status
issues.append(
MessageIssue(
severity="error",
code="missing_sender",
message="No effective From address is configured; Campaign must resolve the sender before building.",
behavior="block",
source="recipients",
)
)
return MessageValidationStatus.BLOCKED
def _render_message_template(
config: CampaignConfig,
campaign_file: str | Path,
@@ -784,6 +803,11 @@ def build_entry_message(
if not entry.active:
return _inactive_entry_message(config=config, entry=entry, entry_index=entry_index, context=context)
context.validation_status = _validate_required_sender(
context.senders,
context.issues,
context.validation_status,
)
context.validation_status = _validate_required_recipients(
config,
context.recipients,

39
src/govoplan_campaign/backend/persistence/campaigns.py
View File

@@ -27,10 +27,14 @@ from govoplan_campaign.backend.db.models import (
JobValidationStatus,
)
from govoplan_campaign.backend.campaign.loader import load_campaign_json, validate_against_schema
from govoplan_campaign.backend.campaign.mail_profile_boundary import (
assert_campaign_uses_mail_profile_reference,
campaign_mail_profile_id,
)
from govoplan_campaign.backend.campaign.validation import validate_campaign_config
from govoplan_campaign.backend.messages.builder import build_campaign_messages
from govoplan_campaign.backend.messages.models import MessageDraft
from govoplan_campaign.backend.sending.execution import create_execution_snapshot
from govoplan_campaign.backend.sending.execution import create_execution_snapshot, profile_transport_revisions
from govoplan_campaign.backend.campaign.models import CampaignConfig
from govoplan_campaign.backend.integrations import files_integration, mail_integration
from govoplan_campaign.backend.path_security import assert_server_safe_campaign_paths
@@ -61,15 +65,26 @@ def load_campaign_config_from_json(
owner_user_id: str | None = None,
owner_group_id: str | None = None,
) -> CampaignConfig:
materialized = mail_integration().materialize_campaign_mail_profile_config(
# Validate the persisted Campaign-to-Mail contract before asking Mail for a
# non-secret capability summary. Campaign never receives resolved transport
# settings, account identities, or credentials.
assert_campaign_uses_mail_profile_reference(raw_json)
validate_against_schema(raw_json)
materialized = copy.deepcopy(raw_json)
profile_id = campaign_mail_profile_id(raw_json)
if profile_id:
summary = mail_integration().campaign_profile_delivery_summary(
session,
tenant_id=tenant_id,
raw_json=raw_json,
campaign_id=campaign_id,
profile_id=profile_id,
owner_user_id=owner_user_id,
owner_group_id=owner_group_id,
)
validate_against_schema(materialized)
materialized.setdefault("server", {})["profile_capabilities"] = {
"smtp_available": bool(summary.get("smtp_available")),
"imap_available": bool(summary.get("imap_available")),
}
return CampaignConfig.model_validate(materialized)
@@ -485,13 +500,19 @@ def build_campaign_version(
[job for job, _message in job_build_pairs],
stage="built",
)
runtime_smtp = managed_config.server.runtime_smtp_config()
if not runtime_smtp:
raise CampaignPersistenceError("Campaign has no SMTP configuration; an execution snapshot cannot be created")
if not managed_config.server.profile_capabilities.smtp_available:
raise CampaignPersistenceError("The selected Mail profile has no SMTP configuration; an execution snapshot cannot be created")
profile_id = campaign_mail_profile_id(version.raw_json if isinstance(version.raw_json, dict) else {})
if profile_id is None:
raise CampaignPersistenceError("Select an authorized Mail profile before building campaign messages")
revisions = profile_transport_revisions(session, version)
if not revisions["smtp"]:
raise CampaignPersistenceError("The selected Mail profile has no SMTP transport revision")
execution_snapshot, execution_snapshot_hash = create_execution_snapshot(
version,
smtp=runtime_smtp,
imap=managed_config.server.runtime_imap_config(),
mail_profile_id=profile_id,
smtp_transport_revision=revisions["smtp"],
imap_transport_revision=revisions["imap"],
delivery=managed_config.delivery,
jobs=[job for job, _message in job_build_pairs],
build_summary=report_json,

73
src/govoplan_campaign/backend/persistence/versions.py
View File

@@ -19,6 +19,12 @@ from govoplan_campaign.backend.db.models import (
JobSendStatus,
)
from govoplan_campaign.backend.sending.execution import clear_execution_snapshot
from govoplan_campaign.backend.campaign.mail_profile_boundary import (
campaign_mail_profile_boundary_violations,
campaign_mail_profile_id,
assert_campaign_uses_mail_profile_reference,
public_campaign_mail_server,
)
from govoplan_campaign.backend.integrations import files_integration, mail_integration
from govoplan_campaign.backend.persistence.campaigns import (
CampaignPersistenceError,
@@ -106,25 +112,7 @@ def minimal_campaign_json(*, external_id: str, name: str, description: str | Non
},
"fields": [],
"global_values": {},
"server": {
"inherit_smtp_credentials": True,
"inherit_imap_credentials": True,
"smtp": {
"host": "",
"port": 587,
"security": "starttls",
},
"imap": {
"host": "",
"port": 993,
"security": "tls",
"sent_folder": "auto",
},
"credentials": {
"smtp": {"username": "", "password": ""}, # nosec B105 - empty draft placeholder.
"imap": {"username": "", "password": ""}, # nosec B105 - empty draft placeholder.
},
},
"server": {},
"recipients": {
"from": [],
"allow_individual_from": False,
@@ -347,6 +335,7 @@ def fork_campaign_version_for_edit(
source_filename: str | None = None,
source_base_path: str | None = None,
autosave: bool = True,
migrate_legacy_mail_settings: bool = False,
) -> CampaignVersion:
"""Create the next sole working version from immutable campaign history.
@@ -371,7 +360,16 @@ def fork_campaign_version_for_edit(
"Create the next working copy from the campaign's current immutable version."
)
base_json = raw_json if raw_json is not None else copy.deepcopy(source.raw_json)
source_json = source.raw_json if isinstance(source.raw_json, dict) else {}
source_requires_mail_migration = bool(campaign_mail_profile_boundary_violations(source_json))
if source_requires_mail_migration and not migrate_legacy_mail_settings:
raise CampaignPersistenceError(
"This version contains legacy campaign-local SMTP/IMAP settings. Create the editable copy from "
"the Mail settings migration action so the audit record is preserved and the copy uses a Mail profile."
)
base_json = raw_json if raw_json is not None else copy.deepcopy(source_json)
if source_requires_mail_migration and raw_json is None:
base_json["server"] = public_campaign_mail_server(source_json)
assert_server_safe_campaign_paths(
base_json,
source_filename=source_filename,
@@ -379,6 +377,7 @@ def fork_campaign_version_for_edit(
managed_files_available=files_integration().available,
)
runtime_json = normalize_campaign_paths(base_json, source_base_path) if source_base_path else copy.deepcopy(base_json)
assert_campaign_uses_mail_profile_reference(runtime_json)
mail_integration().assert_campaign_mail_policy_allows_json(session, tenant_id=tenant_id, raw_json=runtime_json, campaign_id=campaign.id)
new_version = CampaignVersion(
@@ -528,6 +527,7 @@ def update_campaign_version(
source_filename: str | None = None,
source_base_path: str | None = None,
autosave: bool = False,
migrate_legacy_mail_settings: bool = False,
) -> CampaignVersion:
if raw_json is not None or source_filename is not None or source_base_path is not None:
assert_server_safe_campaign_paths(
@@ -547,6 +547,18 @@ def update_campaign_version(
if raw_json is not None:
runtime_json = normalize_campaign_paths(raw_json, source_base_path) if source_base_path else copy.deepcopy(raw_json)
if campaign_mail_profile_boundary_violations(version.raw_json) and not migrate_legacy_mail_settings:
raise CampaignPersistenceError(
"This version contains legacy campaign-local SMTP/IMAP settings. Select an authorized Mail "
"profile on the Mail settings page and explicitly save the migration; the stored legacy version "
"will not be changed automatically."
)
assert_campaign_uses_mail_profile_reference(runtime_json)
if campaign_mail_profile_boundary_violations(version.raw_json) and campaign_mail_profile_id(runtime_json) is None:
raise CampaignPersistenceError(
"Migrating legacy campaign mail settings requires an authorized server.mail_profile_id. "
"Select a Mail profile before saving."
)
mail_integration().assert_campaign_mail_policy_allows_json(session, tenant_id=tenant_id, raw_json=runtime_json, campaign_id=campaign.id)
version.raw_json = runtime_json
version.schema_version = str(runtime_json.get("version", version.schema_version or "1.0"))
@@ -839,6 +851,7 @@ def validate_campaign_partial(raw_json: dict[str, Any], *, section: str | None =
_validate_partial_basics(collector, _dict_value(raw_json, "campaign"))
recipients = _dict_value(raw_json, "recipients")
_validate_partial_sender(collector, recipients)
_validate_partial_mail_profile(collector, raw_json)
_validate_partial_recipients(collector, _dict_value(raw_json, "entries"))
_validate_partial_template(collector, _dict_value(raw_json, "template"))
_validate_partial_attachments(collector, _dict_value(raw_json, "attachments"))
@@ -864,6 +877,26 @@ def _validate_partial_sender(collector: _PartialValidationCollector, recipients:
collector.issue("warning", "sender", "recipients.from.email", "missing_sender_email", "Sender email is not configured yet.")
def _validate_partial_mail_profile(collector: _PartialValidationCollector, raw_json: dict[str, Any]) -> None:
violations = campaign_mail_profile_boundary_violations(raw_json)
if violations:
collector.issue(
"error",
"sender",
"server",
"campaign_local_mail_transport_forbidden",
"Campaign-local SMTP/IMAP settings are not supported. Select or migrate to an authorized Mail-module profile.",
)
elif campaign_mail_profile_id(raw_json) is None:
collector.issue(
"warning",
"sender",
"server.mail_profile_id",
"missing_mail_profile",
"Select an authorized Mail-module profile before validating or delivering this campaign.",
)
def _validate_partial_recipients(collector: _PartialValidationCollector, entries: dict[str, Any]) -> None:
has_inline = bool(entries.get("inline"))
has_source = isinstance(entries.get("source"), dict)

45
src/govoplan_campaign/backend/reports/campaigns.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import csv
import io
import logging
import math
from collections import Counter
from datetime import datetime, timezone
@@ -20,13 +21,20 @@ from govoplan_campaign.backend.db.models import (
SendAttempt,
)
from govoplan_campaign.backend.sending.execution import ExecutionSnapshot
from govoplan_campaign.backend.response_security import public_campaign_payload, public_source_filename
from govoplan_campaign.backend.response_security import (
public_campaign_payload,
public_delivery_result_message,
public_source_filename,
)
class CampaignReportError(RuntimeError):
pass
logger = logging.getLogger(__name__)
def _utcnow_iso() -> str:
return datetime.now(timezone.utc).isoformat()
@@ -88,8 +96,8 @@ def _load_delivery_info(version: CampaignVersion | None, jobs: list[CampaignJob]
"job_count": 0,
"queueable_job_count": 0,
"effective_policy_sha256": None,
"smtp_config_fingerprint": None,
"imap_config_fingerprint": None,
"smtp_transport_revision": None,
"imap_transport_revision": None,
"estimated_remaining_send_seconds": None,
"estimated_remaining_send_human": None,
"background_workers_enabled": bool(core_settings.celery_enabled),
@@ -99,8 +107,9 @@ def _load_delivery_info(version: CampaignVersion | None, jobs: list[CampaignJob]
return default
try:
snapshot = ExecutionSnapshot.model_validate(version.execution_snapshot)
except Exception as exc: # pragma: no cover - reporting should remain available
default["load_error"] = str(exc)
except Exception: # pragma: no cover - reporting should remain available
logger.warning("Campaign execution snapshot could not be loaded for a report")
default["load_error"] = "Execution snapshot is invalid; rebuild the selected version before delivery."
return default
messages_per_minute = snapshot.delivery.rate_limit.messages_per_minute
@@ -131,8 +140,8 @@ def _load_delivery_info(version: CampaignVersion | None, jobs: list[CampaignJob]
"job_count": snapshot.job_count,
"queueable_job_count": snapshot.queueable_job_count,
"effective_policy_sha256": snapshot.effective_policy_sha256,
"smtp_config_fingerprint": snapshot.smtp_config_fingerprint,
"imap_config_fingerprint": snapshot.imap_config_fingerprint,
"smtp_transport_revision": snapshot.smtp_transport_revision,
"imap_transport_revision": snapshot.imap_transport_revision,
"estimated_remaining_send_seconds": estimated_seconds,
"estimated_remaining_send_human": _human_duration(estimated_seconds),
"background_workers_enabled": bool(core_settings.celery_enabled),
@@ -223,7 +232,11 @@ def _recent_failures(jobs: list[CampaignJob], *, limit: int = 20) -> list[dict[s
"send_status": job.send_status,
"imap_status": job.imap_status,
"attempt_count": job.attempt_count,
"last_error": job.last_error,
"last_error": public_delivery_result_message(
last_error=job.last_error,
send_status=job.send_status,
imap_status=job.imap_status,
),
"updated_at": job.updated_at.isoformat() if job.updated_at else None,
}
for job in failed[:limit]
@@ -248,7 +261,11 @@ def _job_row(job: CampaignJob) -> dict[str, Any]:
"smtp_started_at": job.smtp_started_at.isoformat() if job.smtp_started_at else None,
"outcome_unknown_at": job.outcome_unknown_at.isoformat() if job.outcome_unknown_at else None,
"sent_at": job.sent_at.isoformat() if job.sent_at else None,
"last_error": job.last_error,
"last_error": public_delivery_result_message(
last_error=job.last_error,
send_status=job.send_status,
imap_status=job.imap_status,
),
"eml_size_bytes": job.eml_size_bytes,
"eml_sha256": job.eml_sha256,
"issues_count": len(job.issues_snapshot or []),
@@ -333,15 +350,11 @@ def _job_evidence_row(
"latest_smtp_attempt_number": latest_smtp.attempt_number if latest_smtp else None,
"latest_smtp_status": latest_smtp.status if latest_smtp else None,
"latest_smtp_status_code": latest_smtp.smtp_status_code if latest_smtp else None,
"latest_smtp_response": latest_smtp.smtp_response if latest_smtp else None,
"latest_smtp_error_type": latest_smtp.error_type if latest_smtp else None,
"latest_smtp_error_message": latest_smtp.error_message if latest_smtp else None,
"latest_smtp_started_at": latest_smtp.started_at.isoformat() if latest_smtp and latest_smtp.started_at else None,
"latest_smtp_finished_at": latest_smtp.finished_at.isoformat() if latest_smtp and latest_smtp.finished_at else None,
"latest_imap_attempt_number": latest_imap.attempt_number if latest_imap else None,
"latest_imap_status": latest_imap.status if latest_imap else None,
"latest_imap_folder": latest_imap.folder if latest_imap else None,
"latest_imap_error_message": latest_imap.error_message if latest_imap else None,
"latest_imap_created_at": latest_imap.created_at.isoformat() if latest_imap and latest_imap.created_at else None,
"latest_imap_updated_at": latest_imap.updated_at.isoformat() if latest_imap and latest_imap.updated_at else None,
})
@@ -355,7 +368,7 @@ def generate_campaign_report(
campaign_id: str,
version_id: str | None = None,
include_jobs: bool = False,
include_recent_failures: bool = True,
include_recent_failures: bool = False,
) -> dict[str, Any]:
"""Generate a dashboard/report payload for one campaign.
@@ -609,15 +622,11 @@ def generate_jobs_csv(
"latest_smtp_attempt_number",
"latest_smtp_status",
"latest_smtp_status_code",
"latest_smtp_response",
"latest_smtp_error_type",
"latest_smtp_error_message",
"latest_smtp_started_at",
"latest_smtp_finished_at",
"latest_imap_attempt_number",
"latest_imap_status",
"latest_imap_folder",
"latest_imap_error_message",
"latest_imap_created_at",
"latest_imap_updated_at",
]

76
src/govoplan_campaign/backend/reports/emailing.py
View File

@@ -9,11 +9,12 @@ from typing import Any
from sqlalchemy.orm import Session
from govoplan_campaign.backend.db.models import Campaign, CampaignVersion
from govoplan_campaign.backend.campaign.loader import load_campaign_config
from govoplan_campaign.backend.campaign.models import CampaignConfig, SmtpConfig
from govoplan_campaign.backend.persistence.campaigns import _write_campaign_snapshot
from govoplan_campaign.backend.campaign.models import CampaignConfig
from govoplan_campaign.backend.campaign.mail_profile_boundary import campaign_mail_profile_id
from govoplan_campaign.backend.persistence.campaigns import load_version_config
from govoplan_campaign.backend.reports.campaigns import CampaignReportError, generate_campaign_report, generate_jobs_csv
from govoplan_campaign.backend.integrations import SmtpConfigurationError, mail_integration
from govoplan_campaign.backend.integrations import SmtpConfigurationError
from govoplan_campaign.backend.sending.execution import ExecutionSnapshotError, ensure_execution_snapshot
class CampaignReportEmailError(RuntimeError):
@@ -30,8 +31,6 @@ class CampaignReportEmailResult:
sent: bool
attached_jobs_csv: bool
attached_report_json: bool
smtp_host: str | None = None
smtp_port: int | None = None
accepted_count: int | None = None
def as_dict(self) -> dict[str, Any]:
@@ -44,8 +43,6 @@ class CampaignReportEmailResult:
"sent": self.sent,
"attached_jobs_csv": self.attached_jobs_csv,
"attached_report_json": self.attached_report_json,
"smtp_host": self.smtp_host,
"smtp_port": self.smtp_port,
"accepted_count": self.accepted_count,
}
@@ -62,18 +59,15 @@ def _selected_version(
return version
def _load_config(version: CampaignVersion) -> CampaignConfig:
snapshot_path = _write_campaign_snapshot(version)
return load_campaign_config(snapshot_path)
def _load_config(session: Session, version: CampaignVersion) -> CampaignConfig:
_campaign, _version, config = load_version_config(session, version.id)
return config
def _effective_from(config: CampaignConfig) -> tuple[str, str | None]:
if config.recipients.from_:
return config.recipients.from_[0].email, config.recipients.from_[0].name
smtp_config = config.server.runtime_smtp_config()
if smtp_config and smtp_config.username and "@" in smtp_config.username:
return smtp_config.username, None
raise SmtpConfigurationError("Report email requires a recipients.from address or an SMTP username that is an email address")
raise SmtpConfigurationError("Report email requires a Campaign-owned recipients.from address")
def _text_summary(report: dict[str, Any]) -> str:
@@ -150,7 +144,7 @@ def send_campaign_report_email(
version_id: str | None = None,
to: list[str],
include_jobs: bool = False,
attach_jobs_csv: bool = True,
attach_jobs_csv: bool = False,
attach_report_json: bool = False,
dry_run: bool = False,
) -> CampaignReportEmailResult:
@@ -161,10 +155,28 @@ def send_campaign_report_email(
raise CampaignReportEmailError("At least one report recipient is required")
version = _selected_version(session, campaign, version_id)
config = _load_config(version)
smtp_config: SmtpConfig | None = config.server.runtime_smtp_config()
if smtp_config is None:
config = _load_config(session, version)
if not config.server.profile_capabilities.smtp_available:
raise SmtpConfigurationError("Campaign has no SMTP configuration")
profile_id = campaign_mail_profile_id(version.raw_json if isinstance(version.raw_json, dict) else {})
if not profile_id:
raise SmtpConfigurationError("Campaign has no Mail profile reference")
if not isinstance(version.execution_snapshot, dict):
raise CampaignReportEmailError(
"Report email requires a validated and built campaign version with stored Mail-profile evidence."
)
try:
snapshot = ensure_execution_snapshot(session, version)
except ExecutionSnapshotError as exc:
raise CampaignReportEmailError(
"Report email requires a validated and built campaign version with current Mail-profile evidence."
) from exc
if not snapshot.smtp_transport_revision:
raise CampaignReportEmailError("Campaign build evidence has no SMTP transport revision")
if not dry_run:
raise CampaignReportEmailError(
"Report email delivery is disabled until it uses a durable, idempotent Mail-owned outbox with unknown-outcome reconciliation."
)
report = generate_campaign_report(
session,
@@ -172,6 +184,7 @@ def send_campaign_report_email(
campaign_id=campaign_id,
version_id=version.id,
include_jobs=include_jobs,
include_recent_failures=include_jobs,
)
jobs_csv = (
generate_jobs_csv(session, tenant_id=tenant_id, campaign_id=campaign_id, version_id=version.id)
@@ -187,9 +200,6 @@ def send_campaign_report_email(
jobs_csv=jobs_csv,
report_json=report_json,
)
envelope_from, _ = _effective_from(config)
if dry_run:
return CampaignReportEmailResult(
campaign_id=campaign.id,
version_id=version.id,
@@ -199,26 +209,4 @@ def send_campaign_report_email(
sent=False,
attached_jobs_csv=jobs_csv is not None,
attached_report_json=report_json is not None,
smtp_host=smtp_config.host,
smtp_port=smtp_config.port,
)
result = mail_integration().send_email_message(
message,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=to,
)
return CampaignReportEmailResult(
campaign_id=campaign.id,
version_id=version.id,
to=to,
subject=str(message["Subject"]),
dry_run=False,
sent=True,
attached_jobs_csv=jobs_csv is not None,
attached_report_json=report_json is not None,
smtp_host=result.host,
smtp_port=result.port,
accepted_count=result.accepted_count,
)

39
src/govoplan_campaign/backend/response_security.py
View File

@@ -4,6 +4,8 @@ import copy
from pathlib import Path, PureWindowsPath
from typing import Any
from govoplan_campaign.backend.campaign.mail_profile_boundary import public_campaign_mail_server
# These fields locate process-local or storage-backend resources, or authorize
# a worker claim. They are useful for tightly controlled diagnostics but are
@@ -39,6 +41,29 @@ def public_campaign_payload(value: Any) -> Any:
return copy.deepcopy(value)
def public_delivery_result_message(
*,
last_error: Any,
send_status: Any,
imap_status: Any,
) -> str | None:
"""Map persisted provider text to a stable business-safe explanation."""
if not last_error:
return None
clean_send_status = str(send_status or "")
clean_imap_status = str(imap_status or "")
if clean_send_status == "outcome_unknown":
return "SMTP delivery outcome requires operator reconciliation."
if clean_send_status in {"failed_temporary", "failed_permanent"}:
return "SMTP delivery failed; an operator can inspect restricted diagnostics."
if clean_imap_status in {"outcome_unknown", "appending"}:
return "Sent-folder append outcome requires operator reconciliation."
if clean_imap_status in {"failed", "skipped"}:
return "Sent-folder append did not complete; an operator can inspect restricted diagnostics."
return "Delivery recorded a warning; an operator can inspect restricted diagnostics."
def public_campaign_configuration(value: Any) -> Any:
"""Return campaign JSON without infrastructure locators or mail secrets.
@@ -49,18 +74,8 @@ def public_campaign_configuration(value: Any) -> Any:
payload = public_campaign_payload(value)
if not isinstance(payload, dict):
return payload
server = payload.get("server")
if isinstance(server, dict):
for protocol in ("smtp", "imap"):
config = server.get(protocol)
if isinstance(config, dict):
config.pop("password", None)
credentials = server.get("credentials")
if isinstance(credentials, dict):
for protocol in ("smtp", "imap"):
config = credentials.get(protocol)
if isinstance(config, dict):
config.pop("password", None)
if "server" in payload:
payload["server"] = public_campaign_mail_server(payload)
_sanitize_configuration_paths(payload)
return payload

6
src/govoplan_campaign/backend/retention.py
View File

@@ -127,7 +127,11 @@ def _apply_eml_retention(
if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES:
result["skipped_not_final"] += 1
continue
if job.imap_status == JobImapStatus.PENDING.value:
if job.imap_status in {
JobImapStatus.PENDING.value,
JobImapStatus.APPENDING.value,
JobImapStatus.OUTCOME_UNKNOWN.value,
}:
result["skipped_not_final"] += 1
continue
result["eligible"] += 1

84
src/govoplan_campaign/backend/router.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import copy
import dataclasses
import logging
from collections.abc import Callable
from typing import Any
@@ -99,7 +100,10 @@ from govoplan_campaign.backend.db.models import (
)
from govoplan_core.db.session import get_session
from govoplan_campaign.backend.reports.campaigns import CampaignReportError, generate_campaign_report, generate_jobs_csv
from govoplan_campaign.backend.response_security import public_campaign_payload
from govoplan_campaign.backend.response_security import (
public_campaign_payload,
public_delivery_result_message,
)
from govoplan_campaign.backend.reports.emailing import CampaignReportEmailError, send_campaign_report_email
from govoplan_campaign.backend.persistence.campaigns import (
CampaignPersistenceError,
@@ -108,9 +112,15 @@ from govoplan_campaign.backend.persistence.campaigns import (
load_campaign_config_from_json,
validate_campaign_version,
)
from govoplan_campaign.backend.integrations import files_integration
from govoplan_campaign.backend.integrations import (
MailProfileError,
SmtpConfigurationError,
SmtpSendError,
files_integration,
)
from govoplan_campaign.backend.path_security import CampaignPathSecurityError, assert_server_safe_campaign_paths
from govoplan_campaign.backend.campaign.loader import load_campaign_json
from govoplan_campaign.backend.campaign.mail_profile_boundary import campaign_mail_profile_id
from govoplan_campaign.backend.attachments.resolver import resolve_campaign_attachments
from govoplan_core.security.time import utc_now
from govoplan_campaign.backend.persistence.versions import (
@@ -130,6 +140,7 @@ from govoplan_campaign.backend.persistence.versions import (
update_campaign_review_state,
validate_campaign_partial,
)
from govoplan_campaign.backend.dev.mock_campaign import MockCampaignSendError, run_mock_campaign_send
from govoplan_campaign.backend.sending.execution import ExecutionSnapshotError, clear_execution_snapshot
from govoplan_campaign.backend.sending.jobs import (
@@ -147,6 +158,7 @@ from govoplan_campaign.backend.sending.jobs import (
)
router = APIRouter(prefix="/campaigns", tags=["campaigns"])
logger = logging.getLogger(__name__)
CAMPAIGN_JOBS_CURSOR_SCOPE = "campaign.jobs"
CAPABILITY_ADDRESSES_LOOKUP = "addresses.lookup"
@@ -341,9 +353,7 @@ def _recipient_sections_changed(current: dict[str, object] | None, proposed: dic
def _campaign_mail_profile_id(raw_json: dict[str, object] | None) -> str | None:
server = raw_json.get("server") if isinstance(raw_json, dict) else None
value = server.get("mail_profile_id") if isinstance(server, dict) else None
return str(value) if value else None
return campaign_mail_profile_id(raw_json)
def _require_mail_profile_use_if_needed(principal: ApiPrincipal, raw_json: dict[str, object] | None) -> None:
@@ -419,12 +429,14 @@ def _update_campaign_version_detail_response(
source_filename=payload.source_filename,
source_base_path=payload.source_base_path,
autosave=autosave,
migrate_legacy_mail_settings=payload.migrate_legacy_mail_settings,
),
audit_action=audit_action,
details=lambda version: {
"campaign_id": campaign_id,
"current_flow": version.current_flow,
"current_step": version.current_step,
"legacy_mail_settings_migrated": payload.migrate_legacy_mail_settings,
},
validation_error_status=status.HTTP_422_UNPROCESSABLE_CONTENT,
)
@@ -1461,7 +1473,18 @@ def fork_version_for_edit(
"""
payload = payload or CampaignVersionUpdateRequest()
_require_mail_profile_use_if_needed(principal, payload.campaign_json)
source_version = _get_version_for_tenant(session, version_id, principal.tenant_id)
if source_version.campaign_id != campaign_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Campaign version not found")
effective_json = (
payload.campaign_json
if isinstance(payload.campaign_json, dict)
else source_version.raw_json
)
_require_mail_profile_use_if_needed(
principal,
effective_json if isinstance(effective_json, dict) else {},
)
try:
version = fork_campaign_version_for_edit(
session,
@@ -1475,6 +1498,7 @@ def fork_version_for_edit(
source_filename=payload.source_filename,
source_base_path=payload.source_base_path,
autosave=True,
migrate_legacy_mail_settings=payload.migrate_legacy_mail_settings,
)
campaign = _get_campaign_for_tenant(session, campaign_id, principal.tenant_id)
audit_from_principal(
@@ -1483,7 +1507,12 @@ def fork_version_for_edit(
action="campaign.version_forked_for_edit",
object_type="campaign_version",
object_id=version.id,
details={"campaign_id": campaign_id, "source_version_id": version_id, "version_number": version.version_number},
details={
"campaign_id": campaign_id,
"source_version_id": version_id,
"version_number": version.version_number,
"legacy_mail_settings_migrated": payload.migrate_legacy_mail_settings,
},
commit=True,
)
return CampaignCreateResponse(
@@ -1882,7 +1911,11 @@ def _job_summary_payload(
"eml_size_bytes": job.eml_size_bytes,
"eml_sha256": job.eml_sha256,
"attempt_count": job.attempt_count,
"last_error": job.last_error,
"last_error": public_delivery_result_message(
last_error=job.last_error,
send_status=job.send_status,
imap_status=job.imap_status,
),
"queued_at": job.queued_at,
"claimed_at": job.claimed_at,
"smtp_started_at": job.smtp_started_at,
@@ -1925,14 +1958,14 @@ def _job_attempts_payload(
"attempt_number": attempt.attempt_number,
"status": attempt.status,
"smtp_status_code": attempt.smtp_status_code,
"smtp_response": attempt.smtp_response,
"error_type": attempt.error_type,
"error_message": attempt.error_message,
"started_at": attempt.started_at,
"finished_at": attempt.finished_at,
}
if include_diagnostics:
payload["claim_token"] = attempt.claim_token
payload["smtp_response"] = attempt.smtp_response
payload["error_type"] = attempt.error_type
payload["error_message"] = attempt.error_message
smtp_payloads.append(payload)
imap_payloads: list[dict[str, object]] = []
@@ -1942,12 +1975,12 @@ def _job_attempts_payload(
"attempt_number": attempt.attempt_number,
"status": attempt.status,
"folder": attempt.folder,
"error_message": attempt.error_message,
"created_at": attempt.created_at,
"updated_at": attempt.updated_at,
}
if include_diagnostics:
payload["claim_token"] = attempt.claim_token
payload["error_message"] = attempt.error_message
imap_payloads.append(payload)
return {"smtp": smtp_payloads, "imap": imap_payloads}
@@ -1972,6 +2005,7 @@ def _job_diagnostics_payload(
"claimed_at": job.claimed_at,
"smtp_started_at": job.smtp_started_at,
"outcome_unknown_at": job.outcome_unknown_at,
"last_error": job.last_error,
},
attempts=_job_attempts_payload(
send_attempts,
@@ -2587,6 +2621,7 @@ def campaign_summary(
campaign_id=campaign_id,
version_id=version_id,
include_jobs=include_jobs,
include_recent_failures=include_jobs,
)
except CampaignReportError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
@@ -2612,6 +2647,7 @@ def campaign_report(
campaign_id=campaign_id,
version_id=version_id,
include_jobs=include_jobs,
include_recent_failures=include_jobs,
)
except CampaignReportError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
@@ -2650,8 +2686,15 @@ def email_campaign_report(
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("campaigns:report:send")),
):
_get_campaign_for_principal(session, campaign_id, principal)
if payload.include_jobs:
campaign = _get_campaign_for_principal(session, campaign_id, principal)
selected_version_id = payload.version_id or campaign.current_version_id
selected_version = session.get(CampaignVersion, selected_version_id) if selected_version_id else None
if selected_version is not None and selected_version.campaign_id == campaign.id:
_require_mail_profile_use_if_needed(
principal,
selected_version.raw_json if isinstance(selected_version.raw_json, dict) else {},
)
if payload.include_jobs or payload.attach_jobs_csv:
_require_permission(principal, "campaigns:recipient:export")
"""Generate a campaign report and send it to one or more email addresses."""
@@ -2679,8 +2722,14 @@ def email_campaign_report(
return ReportEmailResponse(result=result.as_dict())
except CampaignReportError as exc:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
except (CampaignReportEmailError, Exception) as exc:
except (CampaignReportEmailError, MailProfileError, SmtpConfigurationError, SmtpSendError) as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except Exception as exc:
logger.error("Campaign report email failed with an unexpected internal error")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Campaign report email could not be completed.",
) from exc
@@ -2984,6 +3033,7 @@ def resolve_campaign_job_outcome(
job_id=job_id,
decision=payload.decision,
note=payload.note,
commit=False,
)
audit_from_principal(
session,
@@ -2996,7 +3046,11 @@ def resolve_campaign_job_outcome(
)
return CampaignActionResponse(result=result)
except (QueueingError, ExecutionSnapshotError) as exc:
session.rollback()
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) from exc
except Exception:
session.rollback()
raise
@router.post("/{campaign_id}/mock-send", response_model=MockCampaignSendResponse)

120
src/govoplan_campaign/backend/schema/campaign.schema.json
View File

@@ -90,125 +90,9 @@
"type": "object",
"properties": {
"mail_profile_id": {
"type": "string"
},
"inherit_smtp_credentials": {
"type": "boolean",
"default": true
},
"inherit_imap_credentials": {
"type": "boolean",
"default": true
},
"smtp": {
"type": "object",
"properties": {
"host": {
"type": "string"
},
"port": {
"type": "integer",
"minimum": 1,
"maximum": 65535
},
"username": {
"type": "string"
},
"password": {
"type": "string"
},
"security": {
"type": "string",
"enum": [
"plain",
"tls",
"starttls"
],
"default": "starttls"
},
"timeout_seconds": {
"type": "integer",
"minimum": 1,
"default": 30
}
},
"additionalProperties": false
},
"imap": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean",
"default": false
},
"host": {
"type": "string"
},
"port": {
"type": "integer",
"minimum": 1,
"maximum": 65535
},
"username": {
"type": "string"
},
"password": {
"type": "string"
},
"security": {
"type": "string",
"enum": [
"plain",
"tls",
"starttls"
],
"default": "tls"
},
"sent_folder": {
"type": "string",
"default": "auto"
},
"timeout_seconds": {
"type": "integer",
"minimum": 1,
"default": 30
}
},
"additionalProperties": false
},
"credentials": {
"type": "object",
"properties": {
"smtp": {
"type": "object",
"properties": {
"username": {
"type": "string"
},
"password": {
"type": "string"
}
},
"additionalProperties": false
},
"imap": {
"type": "object",
"properties": {
"username": {
"type": "string"
},
"password": {
"type": "string"
}
},
"additionalProperties": false
}
},
"additionalProperties": false,
"default": {
"smtp": {},
"imap": {}
}
"minLength": 1,
"description": "Stable reference to an authorized profile owned by the Mail module. Campaign JSON never stores SMTP/IMAP settings or credentials."
}
},
"additionalProperties": false

191
src/govoplan_campaign/backend/schemas.py
View File

@@ -6,8 +6,6 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_core.mail.config import ImapConfig, SmtpConfig
from govoplan_campaign.backend.response_security import (
public_campaign_configuration,
public_campaign_payload,
@@ -55,6 +53,7 @@ class CampaignVersionUpdateRequest(BaseModel):
editor_state: dict[str, Any] | None = None
source_filename: str | None = None
source_base_path: str | None = None
migrate_legacy_mail_settings: bool = False
class CampaignVersionSetStepRequest(BaseModel):
@@ -118,6 +117,7 @@ class CampaignVersionResponse(BaseModel):
class CampaignVersionDetailResponse(CampaignVersionResponse):
raw_json: dict[str, Any]
mail_profile_migration_required: bool = False
@field_validator("raw_json", mode="before")
@classmethod
@@ -412,8 +412,21 @@ class CampaignSendJobRequest(BaseModel):
class CampaignResolveOutcomeRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
decision: Literal["smtp_accepted", "not_sent"]
note: str | None = None
decision: Literal[
"smtp_accepted",
"not_sent",
"imap_appended",
"imap_not_appended",
]
note: str | None = Field(default=None, max_length=2000)
@model_validator(mode="after")
def require_imap_reconciliation_evidence(self) -> "CampaignResolveOutcomeRequest":
if self.decision.startswith("imap_"):
self.note = (self.note or "").strip()
if not self.note:
raise ValueError("IMAP reconciliation requires an evidence note")
return self
class ValidateCampaignRequest(BaseModel):
@@ -429,125 +442,6 @@ class BuildCampaignRequest(BaseModel):
write_eml: bool = True
class MailSmtpTestRequest(SmtpConfig):
"""SMTP settings supplied directly from the WebUI mail settings form."""
class MailImapTestRequest(ImapConfig):
"""IMAP settings supplied directly from the WebUI mail settings form."""
MailProfileScope = Literal["system", "tenant", "user", "group", "campaign"]
class MailCredentialPolicyPayload(BaseModel):
model_config = ConfigDict(extra="forbid")
inherit: bool | None = None
allow_override: bool | None = None
class MailProfilePolicyPayload(BaseModel):
model_config = ConfigDict(extra="forbid")
allowed_profile_ids: list[str] = Field(default_factory=list)
allow_user_profiles: bool | None = None
allow_group_profiles: bool | None = None
allow_campaign_profiles: bool | None = None
smtp_credentials: MailCredentialPolicyPayload = Field(default_factory=MailCredentialPolicyPayload)
imap_credentials: MailCredentialPolicyPayload = Field(default_factory=MailCredentialPolicyPayload)
whitelist: dict[str, list[str]] = Field(default_factory=dict)
blacklist: dict[str, list[str]] = Field(default_factory=dict)
class MailProfilePolicyUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
policy: MailProfilePolicyPayload = Field(default_factory=MailProfilePolicyPayload)
class MailProfilePolicyResponse(BaseModel):
scope_type: MailProfileScope
scope_id: str | None = None
policy: dict[str, Any]
effective_policy: dict[str, Any] | None = None
class MailServerProfileCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=255)
slug: str | None = Field(default=None, max_length=100)
description: str | None = None
is_active: bool = True
scope_type: MailProfileScope = "tenant"
scope_id: str | None = None
smtp: SmtpConfig
imap: ImapConfig | None = None
class MailServerProfileUpdateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str | None = Field(default=None, max_length=255)
slug: str | None = Field(default=None, max_length=100)
description: str | None = None
is_active: bool | None = None
smtp: SmtpConfig | None = None
imap: ImapConfig | None = None
clear_imap: bool = False
class MailServerProfileResponse(BaseModel):
id: str
tenant_id: str | None = None
scope_type: MailProfileScope = "tenant"
scope_id: str | None = None
name: str
slug: str
description: str | None = None
is_active: bool
smtp: dict[str, Any]
imap: dict[str, Any] | None = None
smtp_password_configured: bool = False
imap_password_configured: bool = False
created_at: datetime
updated_at: datetime
class MailServerProfileListResponse(BaseModel):
profiles: list[MailServerProfileResponse] = Field(default_factory=list)
class MailConnectionTestResponse(BaseModel):
ok: bool
protocol: Literal["smtp", "imap"]
host: str | None = None
port: int | None = None
security: str | None = None
message: str
details: dict[str, Any] = Field(default_factory=dict)
class MailImapFolderResponse(BaseModel):
name: str
flags: list[str] = Field(default_factory=list)
class MailImapFolderListResponse(BaseModel):
ok: bool
protocol: Literal["imap"] = "imap"
host: str | None = None
port: int | None = None
security: str | None = None
message: str
folders: list[MailImapFolderResponse] = Field(default_factory=list)
detected_sent_folder: str | None = None
details: dict[str, Any] = Field(default_factory=dict)
class ApiKeyCreateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -630,13 +524,60 @@ class CampaignActionResponse(BaseModel):
class ReportEmailRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
to: list[str]
to: list[str] = Field(min_length=1, max_length=50)
version_id: str | None = None
include_jobs: bool = False
attach_jobs_csv: bool = True
attach_jobs_csv: bool = False
attach_report_json: bool = False
dry_run: bool = False
@field_validator("to", mode="before")
@classmethod
def normalize_and_validate_recipients(cls, value: Any) -> Any:
if not isinstance(value, list):
return value
if not 1 <= len(value) <= 50:
raise ValueError("report email requires between 1 and 50 recipients")
recipients: list[str] = []
seen: set[str] = set()
for item in value:
if not isinstance(item, str):
raise ValueError("report recipients must be email-address strings")
recipient = item.strip()
if len(recipient) > 320:
raise ValueError("report recipient addresses must be at most 320 characters")
if any(ord(character) < 32 or ord(character) == 127 for character in recipient):
raise ValueError("report recipient addresses must not contain control characters")
if recipient.count("@") != 1:
raise ValueError("report recipients must be email addresses")
local, domain = recipient.split("@", 1)
if (
not local
or not domain
or any(character.isspace() for character in recipient)
or any(character in ',;:<>[]()\\"' for character in recipient)
or local.startswith(".")
or local.endswith(".")
or ".." in local
or domain.startswith(".")
or domain.endswith(".")
or ".." in domain
or any(
not label
or label.startswith("-")
or label.endswith("-")
or not all(character.isalnum() or character == "-" for character in label)
for label in domain.split(".")
)
):
raise ValueError("report recipients must be email addresses")
key = recipient.casefold()
if key in seen:
continue
seen.add(key)
recipients.append(recipient)
return recipients
class ReportEmailResponse(BaseModel):
result: dict[str, Any]

193
src/govoplan_campaign/backend/sending/execution.py
View File

@@ -9,11 +9,16 @@ from pydantic import BaseModel, ConfigDict
from sqlalchemy.orm import Session
from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion, JobValidationStatus
from govoplan_campaign.backend.campaign.models import DeliveryConfig, ImapConfig, SmtpConfig
from govoplan_campaign.backend.campaign.models import DeliveryConfig
from govoplan_campaign.backend.campaign.mail_profile_boundary import (
CampaignMailProfileBoundaryError,
assert_campaign_uses_mail_profile_reference,
campaign_mail_profile_id,
)
from govoplan_campaign.backend.integrations import MailProfileError, files_integration, mail_integration
from govoplan_campaign.backend.path_security import CampaignPathSecurityError, assert_server_safe_campaign_paths
SNAPSHOT_VERSION = "3"
SNAPSHOT_VERSION = "5"
class ExecutionSnapshotError(RuntimeError):
@@ -25,9 +30,9 @@ class ExecutionSnapshot(BaseModel):
Rendered messages and attachment evidence remain normalized in
``CampaignJob`` and ``CampaignAttachmentUse``. This record freezes the
mutable transport/runtime configuration and cryptographically binds it to
the build and the exact set of persisted jobs without duplicating all
recipient data into one large JSON value.
Mail-profile reference, delivery policy, and opaque transport revisions and
cryptographically binds them to the build and exact persisted jobs. Mail
owns the resolved transport configuration and credentials.
"""
model_config = ConfigDict(extra="forbid")
@@ -35,6 +40,7 @@ class ExecutionSnapshot(BaseModel):
snapshot_version: str = SNAPSHOT_VERSION
campaign_version_id: str
campaign_json_sha256: str
mail_profile_id: str
created_at: str
build_token: str | None = None
built_at: str | None = None
@@ -42,10 +48,8 @@ class ExecutionSnapshot(BaseModel):
queueable_job_count: int = 0
job_manifest_sha256: str | None = None
effective_policy_sha256: str | None = None
smtp_config_fingerprint: str | None = None
imap_config_fingerprint: str | None = None
smtp: SmtpConfig
imap: ImapConfig | None = None
smtp_transport_revision: str | None = None
imap_transport_revision: str | None = None
delivery: DeliveryConfig
@@ -61,110 +65,50 @@ def snapshot_hash(payload: dict[str, Any]) -> str:
return _sha256(payload)
def _transport_fingerprint(config: SmtpConfig | ImapConfig | None) -> str | None:
if config is None:
return None
payload = config.model_dump(mode="json")
# The fingerprint is safe to expose in reports. It identifies the effective
# account/transport settings without incorporating or revealing the secret.
if "password" in payload:
payload["password"] = "<configured>" if payload.get("password") else None
return _sha256(payload)
def _redacted_transport_config(config: SmtpConfig | ImapConfig | None) -> SmtpConfig | ImapConfig | None:
if config is None:
return None
payload = config.model_dump(mode="json")
payload["password"] = None
if isinstance(config, SmtpConfig):
return SmtpConfig.model_validate(payload)
return ImapConfig.model_validate(payload)
def _transport_password_from_campaign_json(raw_json: dict[str, Any] | None, name: str) -> str | None:
server = raw_json.get("server") if isinstance(raw_json, dict) else None
credentials = server.get("credentials") if isinstance(server, dict) and isinstance(server.get("credentials"), dict) else None
config = credentials.get(name) if isinstance(credentials, dict) and isinstance(credentials.get(name), dict) else None
password = config.get("password") if isinstance(config, dict) else None
if password is None:
legacy_config = server.get(name) if isinstance(server, dict) else None
password = legacy_config.get("password") if isinstance(legacy_config, dict) else None
if password is None:
return None
return str(password)
def _server_from_campaign_json(raw_json: dict[str, Any] | None) -> dict[str, Any]:
server = raw_json.get("server") if isinstance(raw_json, dict) else None
return server if isinstance(server, dict) else {}
def _profile_for_version(session: Session, version: CampaignVersion):
def profile_delivery_summary(session: Session, version: CampaignVersion) -> dict[str, Any]:
raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
_assert_version_mail_profile_boundary(raw_json)
mail = mail_integration()
profile_id = mail.mail_profile_id_from_campaign_json(version.raw_json if isinstance(version.raw_json, dict) else {})
if not profile_id:
return None
profile_id = campaign_mail_profile_id(raw_json)
if profile_id is None: # Kept explicit for static typing; the assertion above requires it.
raise ExecutionSnapshotError("Campaign has no Mail profile reference")
campaign = session.get(Campaign, version.campaign_id)
if campaign is None:
raise ExecutionSnapshotError("Campaign not found for mail-server profile resolution")
try:
return mail.ensure_mail_profile_allowed_for_campaign(session, tenant_id=campaign.tenant_id, campaign_id=campaign.id, profile_id=profile_id, require_active=True)
return mail.campaign_profile_delivery_summary(
session,
tenant_id=campaign.tenant_id,
campaign_id=campaign.id,
profile_id=profile_id,
)
except MailProfileError as exc:
raise ExecutionSnapshotError(str(exc)) from exc
def runtime_smtp_config(session: Session, version: CampaignVersion, snapshot: ExecutionSnapshot) -> SmtpConfig:
payload = snapshot.smtp.model_dump(mode="json")
if not payload.get("password"):
server = _server_from_campaign_json(version.raw_json)
profile = _profile_for_version(session, version)
if profile is not None:
campaign = session.get(Campaign, version.campaign_id)
if campaign is None:
raise ExecutionSnapshotError("Campaign not found for mail-server profile resolution")
mail = mail_integration()
profile_payload = mail.smtp_config_from_profile(profile).model_dump(mode="json")
if mail.effective_profile_credentials_inherited(session, tenant_id=campaign.tenant_id, campaign_id=campaign.id, server=server, protocol="smtp"):
return SmtpConfig.model_validate(profile_payload)
return SmtpConfig.model_validate(mail.apply_campaign_credentials(profile_payload, server, "smtp"))
payload["password"] = _transport_password_from_campaign_json(version.raw_json, "smtp")
return SmtpConfig.model_validate(payload)
def profile_transport_revisions(session: Session, version: CampaignVersion) -> dict[str, str | None]:
summary = profile_delivery_summary(session, version)
return {
"smtp": summary.get("smtp_transport_revision"),
"imap": summary.get("imap_transport_revision"),
}
def runtime_imap_config(session: Session, version: CampaignVersion, snapshot: ExecutionSnapshot) -> ImapConfig | None:
server = _server_from_campaign_json(version.raw_json)
if snapshot.imap is None:
profile = _profile_for_version(session, version)
if profile is None:
return None
mail = mail_integration()
imap = mail.imap_config_from_profile(profile)
if imap is None:
return None
campaign = session.get(Campaign, version.campaign_id)
if campaign is None:
raise ExecutionSnapshotError("Campaign not found for mail-server profile resolution")
imap_payload = imap.model_dump(mode="json")
if mail.effective_profile_credentials_inherited(session, tenant_id=campaign.tenant_id, campaign_id=campaign.id, server=server, protocol="imap"):
return ImapConfig.model_validate(imap_payload)
return ImapConfig.model_validate(mail.apply_campaign_credentials(imap_payload, server, "imap"))
payload = snapshot.imap.model_dump(mode="json")
if not payload.get("password"):
profile = _profile_for_version(session, version)
if profile is not None:
mail = mail_integration()
imap = mail.imap_config_from_profile(profile)
if imap is not None:
campaign = session.get(Campaign, version.campaign_id)
if campaign is None:
raise ExecutionSnapshotError("Campaign not found for mail-server profile resolution")
imap_payload = imap.model_dump(mode="json")
if mail.effective_profile_credentials_inherited(session, tenant_id=campaign.tenant_id, campaign_id=campaign.id, server=server, protocol="imap"):
return ImapConfig.model_validate(imap_payload)
return ImapConfig.model_validate(mail.apply_campaign_credentials(imap_payload, server, "imap"))
payload["password"] = _transport_password_from_campaign_json(version.raw_json, "imap")
return ImapConfig.model_validate(payload)
def _assert_snapshot_profile_matches_version(version: CampaignVersion, snapshot: ExecutionSnapshot) -> None:
raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
_assert_version_mail_profile_boundary(raw_json)
if campaign_mail_profile_id(raw_json) != snapshot.mail_profile_id:
raise ExecutionSnapshotError(
"The campaign's Mail profile reference differs from the built execution snapshot. "
"Revalidate and rebuild the campaign before delivery."
)
def _assert_version_mail_profile_boundary(raw_json: dict[str, Any]) -> None:
try:
assert_campaign_uses_mail_profile_reference(raw_json, require_profile=True)
except CampaignMailProfileBoundaryError as exc:
raise ExecutionSnapshotError(str(exc)) from exc
def _policy_fingerprint(raw_json: dict[str, Any], delivery: DeliveryConfig) -> str:
@@ -208,8 +152,9 @@ def job_manifest_hash(jobs: Iterable[CampaignJob]) -> str:
def create_execution_snapshot(
version: CampaignVersion,
*,
smtp: SmtpConfig,
imap: ImapConfig | None,
mail_profile_id: str,
smtp_transport_revision: str,
imap_transport_revision: str | None,
delivery: DeliveryConfig,
jobs: Iterable[CampaignJob] = (),
build_summary: dict[str, Any] | None = None,
@@ -218,26 +163,19 @@ def create_execution_snapshot(
job_list = list(jobs)
summary = build_summary if isinstance(build_summary, dict) else {}
queueable_statuses = {JobValidationStatus.READY.value, JobValidationStatus.WARNING.value}
redacted_smtp = _redacted_transport_config(smtp)
redacted_imap = _redacted_transport_config(imap)
if not isinstance(redacted_smtp, SmtpConfig):
raise ExecutionSnapshotError("Redacted SMTP configuration is invalid.")
if redacted_imap is not None and not isinstance(redacted_imap, ImapConfig):
raise ExecutionSnapshotError("Redacted IMAP configuration is invalid.")
payload = ExecutionSnapshot(
campaign_version_id=version.id,
campaign_json_sha256=_sha256(raw_json),
mail_profile_id=mail_profile_id,
build_token=str(summary.get("build_token") or "") or None,
built_at=str(summary.get("built_at") or "") or None,
job_count=len(job_list),
queueable_job_count=sum(1 for job in job_list if job.validation_status in queueable_statuses),
job_manifest_sha256=job_manifest_hash(job_list) if job_list else None,
effective_policy_sha256=_policy_fingerprint(raw_json, delivery),
smtp_config_fingerprint=_transport_fingerprint(smtp),
imap_config_fingerprint=_transport_fingerprint(imap),
smtp_transport_revision=smtp_transport_revision,
imap_transport_revision=imap_transport_revision,
created_at=datetime.now(timezone.utc).isoformat(),
smtp=redacted_smtp,
imap=redacted_imap,
delivery=delivery,
).model_dump(mode="json")
return payload, snapshot_hash(payload)
@@ -251,27 +189,38 @@ def ensure_execution_snapshot(session: Session, version: CampaignVersion) -> Exe
without a manual data migration.
"""
raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
try:
assert_server_safe_campaign_paths(
version.raw_json if isinstance(version.raw_json, dict) else {},
raw_json,
managed_files_available=files_integration().available,
)
except CampaignPathSecurityError as exc:
raise ExecutionSnapshotError(str(exc)) from exc
_assert_version_mail_profile_boundary(raw_json)
if isinstance(version.execution_snapshot, dict):
if str(version.execution_snapshot.get("snapshot_version") or "") != SNAPSHOT_VERSION:
raise ExecutionSnapshotError(
"This campaign has a legacy execution snapshot that may contain campaign-owned transport data. "
"It is preserved for audit only and cannot be delivered; select a Mail profile, then revalidate "
"and rebuild a new campaign version."
)
snapshot = ExecutionSnapshot.model_validate(version.execution_snapshot)
expected = snapshot_hash(snapshot.model_dump(mode="json"))
if version.execution_snapshot_hash and version.execution_snapshot_hash != expected:
raise ExecutionSnapshotError("Execution snapshot checksum mismatch")
_assert_snapshot_profile_matches_version(version, snapshot)
return snapshot
from govoplan_campaign.backend.persistence.campaigns import load_version_config
_, _, config = load_version_config(session, version.id)
runtime_smtp = config.server.runtime_smtp_config()
if not runtime_smtp:
raise ExecutionSnapshotError("Campaign has no SMTP configuration")
profile_id = campaign_mail_profile_id(raw_json)
if not config.server.profile_capabilities.smtp_available:
raise ExecutionSnapshotError("The selected Mail profile has no SMTP configuration")
if profile_id is None:
raise ExecutionSnapshotError("Campaign has no Mail profile reference")
jobs = (
session.query(CampaignJob)
.filter(CampaignJob.campaign_version_id == version.id)
@@ -280,10 +229,14 @@ def ensure_execution_snapshot(session: Session, version: CampaignVersion) -> Exe
)
if not jobs:
raise ExecutionSnapshotError("Campaign version has no built jobs; rebuild it before delivery")
revisions = profile_transport_revisions(session, version)
if not revisions["smtp"]:
raise ExecutionSnapshotError("The selected Mail profile has no SMTP transport revision")
payload, digest = create_execution_snapshot(
version,
smtp=runtime_smtp,
imap=config.server.runtime_imap_config(),
mail_profile_id=profile_id,
smtp_transport_revision=revisions["smtp"],
imap_transport_revision=revisions["imap"],
delivery=config.delivery,
jobs=jobs,
build_summary=version.build_summary if isinstance(version.build_summary, dict) else {},

573
src/govoplan_campaign/backend/sending/jobs.py
View File

@@ -28,7 +28,7 @@ from govoplan_campaign.backend.db.models import (
ImapAppendAttempt,
SendAttempt,
)
from govoplan_campaign.backend.sending.execution import ExecutionSnapshot, ExecutionSnapshotError, ensure_execution_snapshot, runtime_imap_config, runtime_smtp_config
from govoplan_campaign.backend.sending.execution import ExecutionSnapshot, ExecutionSnapshotError, ensure_execution_snapshot
from govoplan_campaign.backend.runtime import get_registry
from govoplan_campaign.backend.integrations import (
ImapAppendError,
@@ -1040,18 +1040,29 @@ def reconcile_job_outcome(
job_id: str,
decision: str,
note: str | None = None,
commit: bool = True,
) -> dict[str, Any]:
"""Record an operator decision for an uncertain/incomplete worker state.
``smtp_accepted`` protects the message from retry and enables IMAP append.
``not_sent`` converts it into a failed-temporary candidate; retry remains a
separate explicit action.
separate explicit action. ``imap_appended`` completes an uncertain append;
only the evidence-backed ``imap_not_appended`` decision makes it retryable.
"""
campaign = _get_campaign_for_tenant(session, campaign_id=campaign_id, tenant_id=tenant_id)
job = session.get(CampaignJob, job_id)
if not job or job.tenant_id != tenant_id or job.campaign_id != campaign.id:
raise QueueingError("Campaign job not found or not accessible")
if decision in {"imap_appended", "imap_not_appended"}:
return _reconcile_imap_append_outcome(
session,
campaign=campaign,
job=job,
decision=decision,
note=note,
commit=commit,
)
if job.send_status not in {
JobSendStatus.OUTCOME_UNKNOWN.value,
JobSendStatus.SENDING.value,
@@ -1089,7 +1100,10 @@ def reconcile_job_outcome(
session.add(attempt)
session.add(job)
_update_campaign_after_job(session, campaign.id, version.id)
if commit:
session.commit()
else:
session.flush()
return {
"campaign_id": campaign.id,
"version_id": version.id,
@@ -1101,6 +1115,70 @@ def reconcile_job_outcome(
}
def _reconcile_imap_append_outcome(
session: Session,
*,
campaign: Campaign,
job: CampaignJob,
decision: str,
note: str | None,
commit: bool,
) -> dict[str, Any]:
"""Resolve an uncertain append while retaining its immutable attempt row."""
evidence_note = (note or "").strip()
if not evidence_note:
raise QueueingError("IMAP reconciliation requires an evidence note")
if job.send_status not in SMTP_ACCEPTED_STATUSES:
raise QueueingError("Only an SMTP-accepted job can reconcile a Sent-folder append")
if job.imap_status not in {
JobImapStatus.OUTCOME_UNKNOWN.value,
JobImapStatus.APPENDING.value,
}:
raise QueueingError(f"IMAP status {job.imap_status} does not require reconciliation")
attempt = (
session.query(ImapAppendAttempt)
.filter(ImapAppendAttempt.job_id == job.id)
.order_by(ImapAppendAttempt.attempt_number.desc())
.first()
)
if decision == "imap_appended":
job.imap_status = JobImapStatus.APPENDED.value
attempt_status = "reconciled_imap_appended"
files_integration().mark_job_attachment_uses_sent(session, job)
elif decision == "imap_not_appended":
# FAILED is deliberately retryable only after this explicit,
# evidence-backed operator decision.
job.imap_status = JobImapStatus.FAILED.value
attempt_status = "reconciled_imap_not_appended"
else: # Kept defensive for non-HTTP callers.
raise QueueingError("IMAP decision must be 'imap_appended' or 'imap_not_appended'")
job.imap_claimed_at = None
job.imap_claim_token = None
job.last_error = evidence_note
if attempt is not None:
attempt.status = attempt_status
attempt.error_message = evidence_note
session.add(attempt)
session.add(job)
if commit:
session.commit()
else:
session.flush()
return {
"campaign_id": campaign.id,
"version_id": job.campaign_version_id,
"job_id": job.id,
"channel": "imap",
"decision": decision,
"send_status": job.send_status,
"imap_status": job.imap_status,
"note": evidence_note,
}
def _verify_eml_evidence(job: CampaignJob, payload: bytes) -> None:
if job.eml_size_bytes is not None and len(payload) != job.eml_size_bytes:
raise SendJobError(
@@ -1143,7 +1221,7 @@ def _addresses_from_job(job: CampaignJob, field: str) -> list[str]:
return [item.get("email") for item in values if isinstance(item, dict) and item.get("email")]
def _sender_from_job(job: CampaignJob, snapshot: ExecutionSnapshot) -> str:
def _sender_from_job(job: CampaignJob) -> str:
data = job.resolved_recipients or {}
bounce_to = _addresses_from_job(job, "bounce_to")
if bounce_to:
@@ -1151,17 +1229,17 @@ def _sender_from_job(job: CampaignJob, snapshot: ExecutionSnapshot) -> str:
from_data = data.get("from") if isinstance(data, dict) else None
if isinstance(from_data, dict) and from_data.get("email"):
return from_data["email"]
if snapshot.smtp.username:
return snapshot.smtp.username
raise SmtpConfigurationError("No envelope sender could be determined")
raise SmtpConfigurationError(
"No envelope sender could be determined from Campaign-owned recipient data; configure recipients.from or bounce_to before building"
)
def _from_header_from_job(job: CampaignJob, snapshot: ExecutionSnapshot) -> str | None:
def _from_header_from_job(job: CampaignJob) -> str | None:
data = job.resolved_recipients or {}
from_data = data.get("from") if isinstance(data, dict) else None
if isinstance(from_data, dict) and from_data.get("email"):
return str(from_data["email"])
return snapshot.smtp.username
return None
def _recipients_from_job(job: CampaignJob) -> list[str]:
@@ -1443,7 +1521,7 @@ def _send_job_delivery_context(session: Session, job: CampaignJob) -> _SendJobDe
raise SendJobError(str(exc)) from exc
message_bytes = _load_eml_bytes_for_job(job)
envelope_from = _sender_from_job(job, snapshot)
envelope_from = _sender_from_job(job)
envelope_recipients = _recipients_from_job(job)
if not envelope_recipients:
raise SmtpConfigurationError("No envelope recipients could be determined")
@@ -1504,30 +1582,19 @@ def _send_claimed_campaign_job(
)
attempt = _record_attempt_start(session, job, claim_token)
try:
smtp_config = runtime_smtp_config(session, context.version, context.snapshot)
mail_integration().assert_mail_policy_allows_send(
result = mail_integration().send_campaign_email_bytes(
session,
tenant_id=job.tenant_id,
campaign_id=job.campaign_id,
smtp=smtp_config,
envelope_sender=context.envelope_from,
from_header=_from_header_from_job(job, context.snapshot),
recipients=context.envelope_recipients,
)
result = mail_integration().send_email_bytes(
context.message_bytes,
smtp_config=smtp_config,
profile_id=context.snapshot.mail_profile_id,
message_bytes=context.message_bytes,
envelope_from=context.envelope_from,
envelope_recipients=context.envelope_recipients,
from_header=_from_header_from_job(job),
expected_smtp_transport_revision=context.snapshot.smtp_transport_revision or "",
)
return _record_smtp_send_success(
session,
job=job,
attempt=attempt,
snapshot=context.snapshot,
result=result,
enqueue_imap_task=enqueue_imap_task,
)
if result.accepted_count <= 0:
raise SmtpSendError("SMTP did not accept any envelope recipients", temporary=False)
except SmtpSendError as exc:
outcome_unknown = _record_smtp_send_error(session, job=job, attempt=attempt, exc=exc)
if outcome_unknown is not None:
@@ -1536,6 +1603,41 @@ def _send_claimed_campaign_job(
except (MailProfileError, SmtpConfigurationError, SendJobError, OSError) as exc:
_record_permanent_send_error(session, job=job, attempt=attempt, exc=exc)
raise
try:
success = _record_smtp_send_success(
session,
job=job,
attempt=attempt,
snapshot=context.snapshot,
result=result,
)
except Exception:
session.rollback()
current = session.get(CampaignJob, job.id)
if current is None:
raise SendJobError("SMTP accepted the message, but its durable Campaign job record is unavailable.") from None
return mark_job_outcome_unknown(
session,
current,
reason=(
"SMTP accepted the provider effect, but persisting the accepted outcome failed. "
"Automatic retry is stopped until an operator reconciles the job."
),
)
if enqueue_imap_task and _celery_enabled() and job.imap_status == JobImapStatus.PENDING.value:
try:
_celery_enqueue_append_sent_job(job.id)
except Exception:
return SendJobResult(
job_id=success.job_id,
status=success.status,
attempt_number=success.attempt_number,
message=(
f"{success.message + ' ' if success.message else ''}"
"SMTP acceptance was persisted, but the Sent-folder append could not be enqueued; it remains pending."
),
)
return success
def _record_smtp_send_success(
@@ -1545,10 +1647,7 @@ def _record_smtp_send_success(
attempt: SendAttempt,
snapshot: ExecutionSnapshot,
result: object,
enqueue_imap_task: bool,
) -> SendJobResult:
if result.accepted_count <= 0:
raise SmtpSendError("SMTP did not accept any envelope recipients", temporary=False)
refused_warning = _refused_recipient_warning(result)
attempt.finished_at = _utcnow()
attempt.status = "smtp_accepted_with_refusals" if refused_warning else JobSendStatus.SMTP_ACCEPTED.value
@@ -1565,8 +1664,6 @@ def _record_smtp_send_success(
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
if enqueue_imap_task and _celery_enabled() and job.imap_status == JobImapStatus.PENDING.value:
_celery_enqueue_append_sent_job(job.id)
return SendJobResult(
job_id=job.id,
status=JobSendStatus.SMTP_ACCEPTED.value,
@@ -1639,14 +1736,54 @@ def _record_permanent_send_error(
session.commit()
def _record_imap_attempt_start(session: Session, job: CampaignJob) -> ImapAppendAttempt:
def _imap_attempt_count(session: Session, job_id: str) -> int:
return session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job_id).count()
def _claim_job_for_imap_append(session: Session, job: CampaignJob) -> str | None:
"""Atomically grant one worker permission to invoke the IMAP provider."""
claim_token = str(uuid4())
changed = (
session.query(CampaignJob)
.filter(
CampaignJob.id == job.id,
CampaignJob.send_status.in_(list(SMTP_ACCEPTED_STATUSES)),
CampaignJob.imap_status.in_([JobImapStatus.PENDING.value, JobImapStatus.FAILED.value]),
CampaignJob.imap_claim_token.is_(None),
)
.update(
{
CampaignJob.imap_status: JobImapStatus.APPENDING.value,
CampaignJob.imap_claimed_at: _utcnow(),
CampaignJob.imap_claim_token: claim_token,
CampaignJob.last_error: None,
},
synchronize_session=False,
)
)
session.commit()
session.expire_all()
return claim_token if changed == 1 else None
def _record_imap_attempt_start(
session: Session,
job: CampaignJob,
claim_token: str,
) -> ImapAppendAttempt:
if (
job.imap_status != JobImapStatus.APPENDING.value
or job.imap_claim_token != claim_token
):
raise SendJobError("The IMAP append claim was lost before the provider effect started")
existing_count = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count()
attempt = ImapAppendAttempt(
job_id=job.id,
attempt_number=existing_count + 1,
status="running",
claim_token=claim_token,
)
job.imap_status = JobImapStatus.PENDING.value
job.last_error = None
session.add(attempt)
session.add(job)
@@ -1654,13 +1791,241 @@ def _record_imap_attempt_start(session: Session, job: CampaignJob) -> ImapAppend
return attempt
def _effective_imap_folder(snapshot: ExecutionSnapshot) -> str:
delivery_folder = snapshot.delivery.imap_append_sent.folder
if delivery_folder and delivery_folder != "auto":
return delivery_folder
if snapshot.imap and snapshot.imap.sent_folder and snapshot.imap.sent_folder != "auto":
return snapshot.imap.sent_folder
return "auto"
def _imap_blocked_result(
session: Session,
job: CampaignJob,
*,
dry_run: bool,
) -> AppendSentResult | None:
if job.imap_status not in {
JobImapStatus.NOT_REQUESTED.value,
JobImapStatus.APPENDED.value,
JobImapStatus.SKIPPED.value,
JobImapStatus.OUTCOME_UNKNOWN.value,
JobImapStatus.APPENDING.value,
}:
return None
attempts = _imap_attempt_count(session, job.id)
if job.imap_status == JobImapStatus.NOT_REQUESTED.value:
return AppendSentResult(job_id=job.id, status="not_requested", attempt_number=attempts, dry_run=dry_run)
if job.imap_status == JobImapStatus.APPENDED.value:
return AppendSentResult(job_id=job.id, status="already_appended", attempt_number=attempts, dry_run=dry_run)
if job.imap_status == JobImapStatus.SKIPPED.value:
return AppendSentResult(job_id=job.id, status="skipped", attempt_number=attempts, dry_run=dry_run)
if job.imap_status == JobImapStatus.OUTCOME_UNKNOWN.value:
return AppendSentResult(
job_id=job.id,
status=JobImapStatus.OUTCOME_UNKNOWN.value,
attempt_number=attempts,
dry_run=dry_run,
message="The prior IMAP append outcome is unresolved; inspect and reconcile the mailbox before retrying.",
)
if job.imap_status == JobImapStatus.APPENDING.value:
return AppendSentResult(
job_id=job.id,
status="append_in_progress",
attempt_number=attempts,
dry_run=dry_run,
message="Another worker owns the IMAP append claim; automatic duplicate append was stopped.",
)
return None
def _imap_result_after_lost_claim(
session: Session,
*,
job_id: str,
attempt: ImapAppendAttempt,
provider_succeeded: bool,
) -> AppendSentResult:
session.rollback()
current_attempt = session.get(ImapAppendAttempt, attempt.id)
if current_attempt is not None:
current_attempt.status = "late_ack_ignored" if provider_succeeded else "claim_lost"
current_attempt.error_message = (
"The provider returned after this attempt no longer owned the durable IMAP claim."
)
session.add(current_attempt)
if provider_succeeded:
# If no other worker is active, freeze any retryable state. A provider
# success received after claim loss must never be followed by an
# automatic duplicate append.
session.query(CampaignJob).filter(
CampaignJob.id == job_id,
CampaignJob.imap_status.in_(
[
JobImapStatus.PENDING.value,
JobImapStatus.FAILED.value,
JobImapStatus.OUTCOME_UNKNOWN.value,
]
),
CampaignJob.imap_claim_token.is_(None),
).update(
{
CampaignJob.imap_status: JobImapStatus.OUTCOME_UNKNOWN.value,
CampaignJob.last_error: (
"An IMAP provider acknowledgement arrived after the durable claim was lost; operator reconciliation is required."
),
},
synchronize_session=False,
)
session.commit()
session.expire_all()
current = session.get(CampaignJob, job_id)
if current is None:
raise SendJobError("Campaign job disappeared while reconciling a lost IMAP claim")
blocked = _imap_blocked_result(session, current, dry_run=False)
if blocked is not None:
return blocked
return AppendSentResult(
job_id=current.id,
status="claim_lost",
attempt_number=_imap_attempt_count(session, current.id),
message="The IMAP append claim changed while the provider operation was in progress.",
)
def _record_imap_append_failure(
session: Session,
*,
job: CampaignJob,
attempt: ImapAppendAttempt,
claim_token: str,
folder: str,
message: str,
outcome_unknown: bool,
) -> bool:
next_status = (
JobImapStatus.OUTCOME_UNKNOWN.value
if outcome_unknown
else JobImapStatus.FAILED.value
)
changed = (
session.query(CampaignJob)
.filter(
CampaignJob.id == job.id,
CampaignJob.imap_status == JobImapStatus.APPENDING.value,
CampaignJob.imap_claim_token == claim_token,
)
.update(
{
CampaignJob.imap_status: next_status,
CampaignJob.imap_claimed_at: None,
CampaignJob.imap_claim_token: None,
CampaignJob.last_error: message,
},
synchronize_session=False,
)
)
attempt.status = next_status if outcome_unknown else "failed"
attempt.folder = None if folder == "auto" else folder
attempt.error_message = message
session.add(attempt)
session.commit()
session.expire_all()
return changed == 1
def _record_imap_append_success(
session: Session,
*,
job: CampaignJob,
attempt: ImapAppendAttempt,
claim_token: str,
folder: str,
) -> AppendSentResult:
changed = (
session.query(CampaignJob)
.filter(
CampaignJob.id == job.id,
CampaignJob.imap_status == JobImapStatus.APPENDING.value,
CampaignJob.imap_claim_token == claim_token,
)
.update(
{
CampaignJob.imap_status: JobImapStatus.APPENDED.value,
CampaignJob.imap_claimed_at: None,
CampaignJob.imap_claim_token: None,
CampaignJob.last_error: None,
},
synchronize_session=False,
)
)
if changed != 1:
return _imap_result_after_lost_claim(
session,
job_id=job.id,
attempt=attempt,
provider_succeeded=True,
)
attempt.status = "appended"
attempt.folder = folder
attempt.error_message = None
session.add(attempt)
files_integration().mark_job_attachment_uses_sent(session, job)
session.commit()
session.expire_all()
return AppendSentResult(
job_id=job.id,
status="appended",
attempt_number=attempt.attempt_number,
folder=folder,
)
def _mark_imap_append_outcome_unknown_after_effect(
session: Session,
*,
job_id: str,
attempt_id: str,
claim_token: str,
reason: str,
) -> AppendSentResult:
"""Freeze retry after a provider effect whose local persistence failed."""
session.rollback()
current_attempt = session.get(ImapAppendAttempt, attempt_id)
if current_attempt is not None:
current_attempt.status = JobImapStatus.OUTCOME_UNKNOWN.value
current_attempt.error_message = reason
session.add(current_attempt)
changed = (
session.query(CampaignJob)
.filter(
CampaignJob.id == job_id,
CampaignJob.imap_status == JobImapStatus.APPENDING.value,
CampaignJob.imap_claim_token == claim_token,
)
.update(
{
CampaignJob.imap_status: JobImapStatus.OUTCOME_UNKNOWN.value,
CampaignJob.imap_claimed_at: None,
CampaignJob.imap_claim_token: None,
CampaignJob.last_error: reason,
},
synchronize_session=False,
)
)
if changed != 1:
if current_attempt is not None:
session.rollback()
current_attempt = session.get(ImapAppendAttempt, attempt_id)
if current_attempt is None:
raise SendJobError("IMAP append outcome is unknown and its attempt record is unavailable")
return _imap_result_after_lost_claim(
session,
job_id=job_id,
attempt=current_attempt,
provider_succeeded=True,
)
session.commit()
session.expire_all()
return AppendSentResult(
job_id=job_id,
status=JobImapStatus.OUTCOME_UNKNOWN.value,
attempt_number=_imap_attempt_count(session, job_id),
message=reason,
)
def append_sent_for_job(session: Session, *, job_id: str, dry_run: bool = False) -> AppendSentResult:
@@ -1671,14 +2036,9 @@ def append_sent_for_job(session: Session, *, job_id: str, dry_run: bool = False)
raise SendJobError(f"Job not found: {job_id}")
if job.send_status not in SMTP_ACCEPTED_STATUSES:
return AppendSentResult(job_id=job_id, status="not_sent", attempt_number=0, dry_run=dry_run, message="SMTP has not accepted this job")
if job.imap_status == JobImapStatus.NOT_REQUESTED.value:
return AppendSentResult(job_id=job_id, status="not_requested", attempt_number=0, dry_run=dry_run)
if job.imap_status == JobImapStatus.APPENDED.value:
attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count()
return AppendSentResult(job_id=job_id, status="already_appended", attempt_number=attempts, dry_run=dry_run)
if job.imap_status == JobImapStatus.SKIPPED.value:
attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count()
return AppendSentResult(job_id=job_id, status="skipped", attempt_number=attempts, dry_run=dry_run)
blocked = _imap_blocked_result(session, job, dry_run=dry_run)
if blocked is not None:
return blocked
version = session.get(CampaignVersion, job.campaign_version_id)
if not version:
@@ -1692,16 +2052,15 @@ def append_sent_for_job(session: Session, *, job_id: str, dry_run: bool = False)
session.add(job)
session.commit()
return AppendSentResult(job_id=job.id, status="not_requested", attempt_number=0, dry_run=dry_run)
imap_config = runtime_imap_config(session, version, snapshot)
if not imap_config:
if not snapshot.imap_transport_revision:
job.imap_status = JobImapStatus.SKIPPED.value
job.last_error = "IMAP append requested, but the execution snapshot has no IMAP configuration"
job.last_error = "IMAP append requested, but the selected Mail profile has no IMAP configuration"
session.add(job)
session.commit()
return AppendSentResult(job_id=job.id, status="skipped", attempt_number=0, dry_run=dry_run, message=job.last_error)
message_bytes = _load_eml_bytes_for_job(job)
folder = _effective_imap_folder(snapshot)
folder = snapshot.delivery.imap_append_sent.folder or "auto"
if dry_run:
attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id == job.id).count()
return AppendSentResult(
@@ -1713,39 +2072,95 @@ def append_sent_for_job(session: Session, *, job_id: str, dry_run: bool = False)
message=f"Would append {len(message_bytes)} bytes to IMAP folder {folder!r}",
)
attempt = _record_imap_attempt_start(session, job)
claim_token = _claim_job_for_imap_append(session, job)
if claim_token is None:
current = session.get(CampaignJob, job.id)
if current is None:
raise SendJobError(f"Job disappeared while claiming IMAP append: {job.id}")
blocked = _imap_blocked_result(session, current, dry_run=False)
if blocked is not None:
return blocked
return AppendSentResult(
job_id=current.id,
status="not_claimed",
attempt_number=_imap_attempt_count(session, current.id),
message=f"Job is no longer eligible for IMAP append: {current.imap_status}",
)
job = session.get(CampaignJob, job.id)
if job is None:
raise SendJobError("Claimed campaign job disappeared before IMAP append")
attempt = _record_imap_attempt_start(session, job, claim_token)
try:
mail_integration().assert_mail_policy_allows_send(
result = mail_integration().append_campaign_message_to_sent(
session,
tenant_id=job.tenant_id,
campaign_id=job.campaign_id,
smtp=snapshot.smtp,
imap=imap_config,
)
result = mail_integration().append_message_to_sent(
message_bytes,
imap_config=imap_config,
profile_id=snapshot.mail_profile_id,
message_bytes=message_bytes,
folder=None if folder == "auto" else folder,
expected_smtp_transport_revision=snapshot.smtp_transport_revision or "",
expected_imap_transport_revision=snapshot.imap_transport_revision,
)
except (MailProfileError, ImapConfigurationError, ImapAppendError) as exc:
outcome_unknown = bool(getattr(exc, "outcome_unknown", False))
owned = _record_imap_append_failure(
session,
job=job,
attempt=attempt,
claim_token=claim_token,
folder=folder,
message=str(exc),
outcome_unknown=outcome_unknown,
)
if not owned:
_imap_result_after_lost_claim(
session,
job_id=job.id,
attempt=attempt,
provider_succeeded=outcome_unknown,
)
attempt.status = "appended"
attempt.folder = result.folder
job.imap_status = JobImapStatus.APPENDED.value
job.last_error = None
files_integration().mark_job_attachment_uses_sent(session, job)
session.add(attempt)
session.add(job)
session.commit()
return AppendSentResult(job_id=job.id, status="appended", attempt_number=attempt.attempt_number, folder=result.folder)
except (MailProfileError, ImapConfigurationError, ImapAppendError, SendJobError, OSError) as exc:
attempt.status = "failed"
attempt.folder = None if folder == "auto" else folder
attempt.error_message = str(exc)
job.imap_status = JobImapStatus.FAILED.value
job.last_error = str(exc)
session.add(attempt)
session.add(job)
session.commit()
raise
except Exception:
reason = (
"The Sent-folder append outcome is unknown after an unexpected provider failure; "
"inspect and reconcile the mailbox before retrying."
)
owned = _record_imap_append_failure(
session,
job=job,
attempt=attempt,
claim_token=claim_token,
folder=folder,
message=reason,
outcome_unknown=True,
)
if not owned:
_imap_result_after_lost_claim(
session,
job_id=job.id,
attempt=attempt,
provider_succeeded=True,
)
raise ImapAppendError(reason, outcome_unknown=True) from None
try:
return _record_imap_append_success(
session,
job=job,
attempt=attempt,
claim_token=claim_token,
folder=result.folder,
)
except Exception:
return _mark_imap_append_outcome_unknown_after_effect(
session,
job_id=job.id,
attempt_id=attempt.id,
claim_token=claim_token,
reason=(
"The IMAP provider accepted the append, but persisting the outcome failed. "
"Automatic retry is stopped until an operator reconciles the mailbox."
),
)
def enqueue_pending_imap_appends(

10
tests/test_attachment_building.py
View File

@@ -44,7 +44,10 @@ class CampaignAttachmentBuildTests(unittest.TestCase):
"campaign": {"id": f"no-attachment-{behavior or legacy_allow}", "name": "No attachment policy", "mode": "test"},
"fields": [],
"global_values": {},
"server": {"smtp": {"host": "smtp.example.invalid", "port": 587, "security": "starttls"}},
"server": {
"mail_profile_id": "profile-1",
"profile_capabilities": {"smtp_available": True},
},
"recipients": {"from": {"email": "sender@example.org", "type": "to"}, "allow_individual_to": True},
"template": {"subject": "Subject", "text": "Body"},
"attachments": attachments,
@@ -140,7 +143,10 @@ class CampaignAttachmentBuildTests(unittest.TestCase):
"campaign": {"id": "zip-missing-pattern", "name": "ZIP missing pattern", "mode": "test"},
"fields": [],
"global_values": {},
"server": {"smtp": {"host": "smtp.example.invalid", "port": 587, "security": "starttls"}},
"server": {
"mail_profile_id": "profile-1",
"profile_capabilities": {"smtp_available": True},
},
"recipients": {"from": {"email": "sender@example.org", "type": "to"}, "allow_individual_to": True},
"template": {"subject": "Subject", "text": "Body"},
"attachments": {

52
tests/test_campaign_reports.py
View File

@@ -3,8 +3,14 @@ from __future__ import annotations
from datetime import UTC, datetime
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from govoplan_campaign.backend.reports.campaigns import _job_evidence_row, _latest_by_job_id
from govoplan_campaign.backend.reports.campaigns import (
_job_evidence_row,
_latest_by_job_id,
_load_delivery_info,
generate_campaign_report,
)
def _dt() -> datetime:
@@ -87,9 +93,12 @@ def test_job_evidence_row_contains_transport_and_message_evidence() -> None:
assert "bundle.zip" in row["attachment_names"]
assert "notice.pdf" in row["attachment_names"]
assert row["latest_smtp_status_code"] == 250
assert row["latest_smtp_response"] == "2.0.0 queued"
assert "latest_smtp_response" not in row
assert "latest_smtp_error_type" not in row
assert "latest_smtp_error_message" not in row
assert row["latest_imap_status"] == "appended"
assert row["latest_imap_folder"] == "Sent"
assert "latest_imap_error_message" not in row
assert "eml_storage_key" not in row
assert "eml_local_path" not in row
@@ -107,6 +116,45 @@ def test_latest_by_job_id_keeps_highest_attempt_number() -> None:
assert latest["job-2"].attempt_number == 2
def test_invalid_legacy_snapshot_never_echoes_validation_details() -> None:
version = SimpleNamespace(
execution_snapshot={
"snapshot_version": "legacy",
"smtp": {"host": "smtp.internal.example", "password": "provider-secret"},
},
execution_snapshot_hash=None,
execution_snapshot_at=None,
)
delivery = _load_delivery_info(version, [])
assert delivery["load_error"] == "Execution snapshot is invalid; rebuild the selected version before delivery."
assert "provider-secret" not in repr(delivery)
assert "smtp.internal.example" not in repr(delivery)
def test_aggregate_report_omits_recipient_level_failures_by_default() -> None:
campaign = SimpleNamespace(id="campaign-1")
version = SimpleNamespace(id="version-1")
with (
patch("govoplan_campaign.backend.reports.campaigns._get_campaign", return_value=campaign),
patch("govoplan_campaign.backend.reports.campaigns._selected_version", return_value=version),
patch("govoplan_campaign.backend.reports.campaigns._report_jobs", return_value=[]),
patch(
"govoplan_campaign.backend.reports.campaigns._campaign_report_payload",
return_value={"cards": {}},
) as payload,
):
report = generate_campaign_report(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
)
assert report == {"cards": {}}
assert payload.call_args.kwargs["include_recent_failures"] is False
class CampaignReportProjectionTests(unittest.TestCase):
def test_evidence_projection(self) -> None:
test_job_evidence_row_contains_transport_and_message_evidence()

401
tests/test_imap_append_idempotency.py Normal file
View File

@@ -0,0 +1,401 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import importlib
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from sqlalchemy import create_engine, text
from govoplan_campaign.backend import router
from govoplan_campaign.backend.db.models import (
CampaignJob,
ImapAppendAttempt,
JobImapStatus,
JobSendStatus,
)
from govoplan_campaign.backend.retention import _apply_eml_retention
from govoplan_campaign.backend.schemas import CampaignResolveOutcomeRequest
from govoplan_campaign.backend.sending.jobs import (
AppendSentResult,
QueueingError,
_claim_job_for_imap_append,
_record_imap_append_success,
append_sent_for_job,
reconcile_job_outcome,
)
@pytest.mark.parametrize(
("imap_status", "expected_status"),
[
(JobImapStatus.APPENDING.value, "append_in_progress"),
(JobImapStatus.OUTCOME_UNKNOWN.value, JobImapStatus.OUTCOME_UNKNOWN.value),
],
)
def test_in_progress_and_unknown_jobs_never_reinvoke_mail_provider(
imap_status: str,
expected_status: str,
) -> None:
job = SimpleNamespace(
id="job-1",
send_status=JobSendStatus.SMTP_ACCEPTED.value,
imap_status=imap_status,
)
session = SimpleNamespace(get=lambda _model, _id: job)
with (
patch("govoplan_campaign.backend.sending.jobs._imap_attempt_count", return_value=1),
patch("govoplan_campaign.backend.sending.jobs.mail_integration") as mail,
):
result = append_sent_for_job(
session, # type: ignore[arg-type]
job_id="job-1",
)
assert result.status == expected_status
mail.assert_not_called()
def test_imap_claim_is_a_single_atomic_state_transition() -> None:
session = MagicMock()
query = session.query.return_value
query.filter.return_value.update.return_value = 1
job = SimpleNamespace(id="job-1")
claim_token = _claim_job_for_imap_append(
session,
job, # type: ignore[arg-type]
)
assert claim_token
changes = query.filter.return_value.update.call_args.args[0]
assert changes[CampaignJob.imap_status] == JobImapStatus.APPENDING.value
assert changes[CampaignJob.imap_claim_token] == claim_token
session.commit.assert_called_once_with()
session.expire_all.assert_called_once_with()
def test_late_provider_acknowledgement_cannot_overwrite_a_changed_claim() -> None:
session = MagicMock()
session.query.return_value.filter.return_value.update.return_value = 0
job = SimpleNamespace(id="job-1")
attempt = SimpleNamespace(id="attempt-1", attempt_number=1)
expected = AppendSentResult(
job_id="job-1",
status=JobImapStatus.OUTCOME_UNKNOWN.value,
attempt_number=1,
)
with (
patch(
"govoplan_campaign.backend.sending.jobs._imap_result_after_lost_claim",
return_value=expected,
) as lost_claim,
patch("govoplan_campaign.backend.sending.jobs.files_integration") as files,
):
result = _record_imap_append_success(
session,
job=job, # type: ignore[arg-type]
attempt=attempt, # type: ignore[arg-type]
claim_token="claim-1",
folder="Sent",
)
assert result is expected
lost_claim.assert_called_once_with(
session,
job_id="job-1",
attempt=attempt,
provider_succeeded=True,
)
files.assert_not_called()
def test_post_provider_persistence_failure_freezes_imap_retry() -> None:
job = SimpleNamespace(
id="job-1",
tenant_id="tenant-1",
campaign_id="campaign-1",
campaign_version_id="version-1",
send_status=JobSendStatus.SMTP_ACCEPTED.value,
imap_status=JobImapStatus.PENDING.value,
)
version = SimpleNamespace(id="version-1")
snapshot = SimpleNamespace(
mail_profile_id="profile-1",
smtp_transport_revision="smtp-revision",
imap_transport_revision="imap-revision",
delivery=SimpleNamespace(
imap_append_sent=SimpleNamespace(enabled=True, folder="Sent"),
),
)
attempt = SimpleNamespace(id="attempt-1", attempt_number=1)
class Session:
def get(self, model, _object_id):
return version if model.__name__ == "CampaignVersion" else job
class Mail:
def append_campaign_message_to_sent(self, *_args, **_kwargs):
return SimpleNamespace(folder="Sent")
expected = AppendSentResult(
job_id="job-1",
status=JobImapStatus.OUTCOME_UNKNOWN.value,
attempt_number=1,
)
session = Session()
with (
patch("govoplan_campaign.backend.sending.jobs.ensure_execution_snapshot", return_value=snapshot),
patch("govoplan_campaign.backend.sending.jobs._load_eml_bytes_for_job", return_value=b"message"),
patch("govoplan_campaign.backend.sending.jobs._claim_job_for_imap_append", return_value="claim-1"),
patch("govoplan_campaign.backend.sending.jobs._record_imap_attempt_start", return_value=attempt),
patch("govoplan_campaign.backend.sending.jobs.mail_integration", return_value=Mail()),
patch(
"govoplan_campaign.backend.sending.jobs._record_imap_append_success",
side_effect=OSError("database unavailable"),
),
patch(
"govoplan_campaign.backend.sending.jobs._mark_imap_append_outcome_unknown_after_effect",
return_value=expected,
) as mark_unknown,
):
result = append_sent_for_job(
session, # type: ignore[arg-type]
job_id="job-1",
)
assert result is expected
assert "Automatic retry is stopped" in mark_unknown.call_args.kwargs["reason"]
def test_attempt_numbers_are_unique_per_job_in_the_model_contract() -> None:
constraints = {
constraint.name
for constraint in ImapAppendAttempt.__table__.constraints
}
assert "uq_imap_append_attempts_job_attempt" in constraints
def test_imap_claim_migration_preserves_and_renumbers_duplicate_attempts() -> None:
migration = importlib.import_module(
"govoplan_campaign.backend.migrations.versions.3c4d5e6f8192_v019_imap_append_claim"
)
engine = create_engine("sqlite+pysqlite:///:memory:")
with engine.begin() as connection:
connection.execute(
text(
"CREATE TABLE imap_append_attempts ("
"id VARCHAR(36) PRIMARY KEY, job_id VARCHAR(36) NOT NULL, "
"attempt_number INTEGER NOT NULL, created_at DATETIME NOT NULL)"
)
)
connection.execute(
text(
"INSERT INTO imap_append_attempts "
"(id, job_id, attempt_number, created_at) VALUES "
"('a2', 'job-1', 1, '2026-01-02'), "
"('a1', 'job-1', 1, '2026-01-01'), "
"('b1', 'job-2', 7, '2026-01-01')"
)
)
with patch.object(migration.op, "get_bind", return_value=connection):
migration._renumber_attempts()
rows = connection.execute(
text(
"SELECT id, job_id, attempt_number FROM imap_append_attempts "
"ORDER BY job_id, attempt_number"
)
).all()
assert rows == [
("a1", "job-1", 1),
("a2", "job-1", 2),
("b1", "job-2", 1),
]
def test_imap_reconciliation_requires_an_evidence_note() -> None:
with pytest.raises(ValueError, match="evidence note"):
CampaignResolveOutcomeRequest(
decision="imap_not_appended",
note=" ",
)
@pytest.mark.parametrize(
("decision", "expected_status", "attempt_status"),
[
(
"imap_appended",
JobImapStatus.APPENDED.value,
"reconciled_imap_appended",
),
(
"imap_not_appended",
JobImapStatus.FAILED.value,
"reconciled_imap_not_appended",
),
],
)
def test_imap_reconciliation_preserves_attempt_and_only_not_appended_is_retryable(
decision: str,
expected_status: str,
attempt_status: str,
) -> None:
campaign = SimpleNamespace(id="campaign-1")
job = SimpleNamespace(
id="job-1",
tenant_id="tenant-1",
campaign_id="campaign-1",
campaign_version_id="version-1",
send_status=JobSendStatus.SMTP_ACCEPTED.value,
imap_status=JobImapStatus.OUTCOME_UNKNOWN.value,
imap_claimed_at=datetime.now(timezone.utc),
imap_claim_token="claim-1",
last_error="unknown",
)
attempt = SimpleNamespace(
id="attempt-1",
status=JobImapStatus.OUTCOME_UNKNOWN.value,
error_message="unknown",
)
session = MagicMock()
session.get.return_value = job
session.query.return_value.filter.return_value.order_by.return_value.first.return_value = attempt
with (
patch(
"govoplan_campaign.backend.sending.jobs._get_campaign_for_tenant",
return_value=campaign,
),
patch("govoplan_campaign.backend.sending.jobs.files_integration") as files,
):
result = reconcile_job_outcome(
session,
tenant_id="tenant-1",
campaign_id="campaign-1",
job_id="job-1",
decision=decision,
note="Mailbox UID evidence checked by operator 42.",
commit=False,
)
assert result["channel"] == "imap"
assert job.imap_status == expected_status
assert job.imap_claimed_at is None
assert job.imap_claim_token is None
assert attempt.status == attempt_status
assert attempt.error_message == "Mailbox UID evidence checked by operator 42."
assert attempt in session.add.call_args_list[0].args
session.flush.assert_called_once_with()
session.commit.assert_not_called()
if decision == "imap_appended":
files().mark_job_attachment_uses_sent.assert_called_once_with(session, job)
else:
files().mark_job_attachment_uses_sent.assert_not_called()
def test_imap_reconciliation_rejects_a_retryable_or_completed_state() -> None:
campaign = SimpleNamespace(id="campaign-1")
job = SimpleNamespace(
id="job-1",
tenant_id="tenant-1",
campaign_id="campaign-1",
campaign_version_id="version-1",
send_status=JobSendStatus.SMTP_ACCEPTED.value,
imap_status=JobImapStatus.FAILED.value,
)
session = MagicMock()
session.get.return_value = job
with patch(
"govoplan_campaign.backend.sending.jobs._get_campaign_for_tenant",
return_value=campaign,
):
with pytest.raises(QueueingError, match="does not require reconciliation"):
reconcile_job_outcome(
session,
tenant_id="tenant-1",
campaign_id="campaign-1",
job_id="job-1",
decision="imap_not_appended",
note="Checked mailbox.",
)
@pytest.mark.parametrize(
("decision", "note"),
[
("smtp_accepted", None),
("imap_appended", "Mailbox evidence checked."),
],
)
def test_reconciliation_rolls_back_state_when_audit_fails(
decision: str,
note: str | None,
) -> None:
payload = CampaignResolveOutcomeRequest(decision=decision, note=note) # type: ignore[arg-type]
principal = SimpleNamespace(tenant_id="tenant-1")
session = MagicMock()
def mutate_without_commit(*_args, **kwargs):
assert kwargs["commit"] is False
session.flush()
return {"decision": decision, "job_id": "job-1"}
with (
patch("govoplan_campaign.backend.router._get_campaign_for_principal"),
patch("govoplan_campaign.backend.router._require_permission"),
patch(
"govoplan_campaign.backend.router.reconcile_job_outcome",
side_effect=mutate_without_commit,
),
patch(
"govoplan_campaign.backend.router.audit_from_principal",
side_effect=RuntimeError("audit unavailable"),
),
):
with pytest.raises(RuntimeError, match="audit unavailable"):
router.resolve_campaign_job_outcome(
"campaign-1",
"job-1",
payload,
session=session,
principal=principal, # type: ignore[arg-type]
)
session.commit.assert_not_called()
session.rollback.assert_called_once_with()
def test_retention_preserves_eml_for_unknown_imap_outcome(tmp_path: Path) -> None:
eml_path = tmp_path / "message.eml"
eml_path.write_bytes(b"message")
job = SimpleNamespace(
campaign_id="campaign-1",
updated_at=datetime.now(timezone.utc) - timedelta(days=10),
queue_status="draft",
send_status=JobSendStatus.SMTP_ACCEPTED.value,
imap_status=JobImapStatus.OUTCOME_UNKNOWN.value,
eml_local_path=str(eml_path),
eml_storage_key=None,
)
query = MagicMock()
query.filter.return_value.order_by.return_value.all.return_value = [job]
session = MagicMock()
session.query.return_value = query
policy = SimpleNamespace(generated_eml_retention_days=1)
result = _apply_eml_retention(
session,
dry_run=False,
now=datetime.now(timezone.utc),
policy_for_campaign_id=lambda _campaign_id: policy,
)
assert result["skipped_not_final"] == 1
assert eml_path.exists()
session.add.assert_not_called()

222
tests/test_mail_profile_boundary.py Normal file
View File

@@ -0,0 +1,222 @@
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from fastapi import HTTPException
from govoplan_campaign.backend import router
from govoplan_campaign.backend.campaign.loader import CampaignSchemaError, validate_against_schema
from govoplan_campaign.backend.campaign.mail_profile_boundary import (
CampaignMailProfileBoundaryError,
assert_campaign_uses_mail_profile_reference,
campaign_mail_profile_boundary_violations,
campaign_mail_profile_id,
)
from govoplan_campaign.backend.campaign.models import DeliveryConfig
from govoplan_campaign.backend.persistence.campaigns import CampaignPersistenceError, load_campaign_config_from_json
from govoplan_campaign.backend.persistence.versions import update_campaign_version
from govoplan_campaign.backend.integrations import MailCampaignIntegration
from govoplan_campaign.backend.sending.execution import ExecutionSnapshotError, create_execution_snapshot, ensure_execution_snapshot
from govoplan_campaign.backend.schemas import CampaignVersionUpdateRequest
def _campaign_json(server: dict[str, object] | None = None) -> dict[str, object]:
return {
"version": "1.0",
"campaign": {"id": "campaign-1", "name": "Campaign", "mode": "send"},
"server": server or {},
"recipients": {"from": [{"email": "sender@example.test"}]},
"template": {"subject": "Subject", "text": "Body", "body_mode": "text"},
"entries": {"inline": []},
}
def test_campaign_mail_contract_accepts_only_a_stable_profile_reference() -> None:
raw = _campaign_json({"mail_profile_id": " profile-1 "})
assert_campaign_uses_mail_profile_reference(raw)
assert campaign_mail_profile_id(raw) == "profile-1"
assert campaign_mail_profile_boundary_violations(raw) == ()
def test_mail_profile_documentation_is_classified_for_adaptive_views() -> None:
from govoplan_campaign.backend.manifest import get_manifest
manifest = get_manifest()
topics = {topic.id: topic for topic in manifest.documentation}
workflow = topics["campaigns.mail-profile-user-journey"]
assert workflow.metadata["kind"] == "workflow"
assert workflow.metadata["route"] == "/campaigns/{campaign_id}/mail"
assert workflow.metadata["prerequisites"]
assert workflow.metadata["steps"]
assert workflow.metadata["outcome"]
assert workflow.metadata["verification"]
assert "campaigns.mail-profile-governance" in workflow.metadata["related_topic_ids"]
assert topics["campaigns.mail-profile-governance"].metadata["kind"] == "reference"
assert topics["campaigns.mail-profile-operations"].metadata["kind"] == "reference"
@pytest.mark.parametrize("legacy_key", ["smtp", "imap", "credentials", "inherit_smtp_credentials", "profile_id"])
def test_campaign_mail_contract_rejects_every_legacy_server_field(legacy_key: str) -> None:
raw = _campaign_json({"mail_profile_id": "profile-1", legacy_key: {}})
with pytest.raises(CampaignMailProfileBoundaryError, match="select an authorized Mail profile"):
assert_campaign_uses_mail_profile_reference(raw)
def test_persisted_schema_rejects_inline_transport_even_without_a_secret() -> None:
with pytest.raises(CampaignSchemaError, match="Additional properties are not allowed"):
validate_against_schema(_campaign_json({"smtp": {"host": "smtp.example.test"}}))
def test_loader_rejects_inline_transport_before_optional_mail_summary() -> None:
integration = SimpleNamespace(campaign_profile_delivery_summary=lambda *_args, **_kwargs: pytest.fail("must not resolve"))
with patch("govoplan_campaign.backend.persistence.campaigns.mail_integration", return_value=integration):
with pytest.raises(CampaignMailProfileBoundaryError, match="remove campaign-local SMTP/IMAP settings"):
load_campaign_config_from_json(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
raw_json=_campaign_json({"smtp": {"password": "secret"}}),
)
def test_loader_uses_only_non_secret_mail_profile_capabilities() -> None:
raw = _campaign_json({"mail_profile_id": "profile-1"})
def summary(_session, **kwargs):
assert kwargs["profile_id"] == "profile-1"
return {
"mail_profile_id": "profile-1",
"smtp_available": True,
"imap_available": False,
"smtp_transport_revision": "opaque-smtp",
"imap_transport_revision": None,
# Even a broken/malicious provider cannot inject extra material into
# Campaign's strict in-memory ServerConfig.
"host": "smtp.example.test",
"password": "secret",
}
integration = SimpleNamespace(campaign_profile_delivery_summary=summary)
with patch("govoplan_campaign.backend.persistence.campaigns.mail_integration", return_value=integration):
config = load_campaign_config_from_json(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
raw_json=raw,
)
assert raw["server"] == {"mail_profile_id": "profile-1"}
assert config.server.mail_profile_id == "profile-1"
assert config.server.profile_capabilities.smtp_available is True
assert config.server.profile_capabilities.imap_available is False
assert "smtp.example.test" not in repr(config.server)
assert "secret" not in repr(config.server)
def test_new_execution_snapshot_stores_reference_and_evidence_not_transport_material() -> None:
raw = _campaign_json({"mail_profile_id": "profile-1"})
version = SimpleNamespace(id="version-1", raw_json=raw)
payload, _digest = create_execution_snapshot(
version, # type: ignore[arg-type]
mail_profile_id="profile-1",
smtp_transport_revision="opaque-smtp-evidence",
imap_transport_revision="opaque-imap-evidence",
delivery=DeliveryConfig(),
)
assert payload["snapshot_version"] == "5"
assert payload["mail_profile_id"] == "profile-1"
assert "smtp" not in payload
assert "imap" not in payload
assert payload["smtp_transport_revision"] == "opaque-smtp-evidence"
assert payload["imap_transport_revision"] == "opaque-imap-evidence"
def test_legacy_execution_snapshot_is_preserved_but_fails_closed() -> None:
version = SimpleNamespace(
raw_json=_campaign_json({"mail_profile_id": "profile-1"}),
execution_snapshot={"snapshot_version": "3", "smtp": {"host": "legacy.example.test"}},
execution_snapshot_hash=None,
)
with patch(
"govoplan_campaign.backend.sending.execution.files_integration",
return_value=SimpleNamespace(available=False),
):
with pytest.raises(ExecutionSnapshotError, match="preserved for audit only"):
ensure_execution_snapshot(object(), version) # type: ignore[arg-type]
assert version.execution_snapshot["smtp"]["host"] == "legacy.example.test"
def test_campaign_mail_adapter_does_not_expose_raw_transport_helpers() -> None:
integration = MailCampaignIntegration(SimpleNamespace())
for name in (
"smtp_config_from_profile",
"imap_config_from_profile",
"send_email_bytes",
"send_email_message",
"materialize_campaign_mail_profile_config",
):
assert not hasattr(integration, name)
def test_editing_a_legacy_record_requires_an_explicit_profile_migration() -> None:
legacy_raw = _campaign_json({"smtp": {"host": "smtp.example.test", "password": "secret"}})
version = SimpleNamespace(id="version-1", campaign_id="campaign-1", raw_json=legacy_raw)
campaign = SimpleNamespace(id="campaign-1", current_version_id="version-1")
with (
patch("govoplan_campaign.backend.persistence.versions.get_campaign_version_for_tenant", return_value=version),
patch("govoplan_campaign.backend.persistence.versions._require_campaign", return_value=campaign),
patch("govoplan_campaign.backend.persistence.versions.ensure_current_working_version"),
patch("govoplan_campaign.backend.persistence.versions.is_version_locked", return_value=False),
):
with pytest.raises(CampaignPersistenceError, match="explicitly save the migration"):
update_campaign_version(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
version_id="version-1",
raw_json=_campaign_json({"mail_profile_id": "profile-1"}),
)
assert version.raw_json is legacy_raw
assert legacy_raw["server"]["smtp"]["password"] == "secret" # type: ignore[index]
def test_fork_inherited_profile_requires_mail_profile_use_scope() -> None:
principal = SimpleNamespace(
tenant_id="tenant-1",
user=SimpleNamespace(id="user-1"),
)
campaign = SimpleNamespace(id="campaign-1")
source = SimpleNamespace(
id="version-1",
campaign_id="campaign-1",
raw_json={"server": {"mail_profile_id": "profile-1"}},
)
with (
patch.object(router, "_get_campaign_for_principal", return_value=campaign),
patch.object(router, "_require_permission"),
patch.object(router, "_get_version_for_tenant", return_value=source),
patch.object(router, "has_scope", return_value=False),
patch.object(router, "fork_campaign_version_for_edit") as fork,
):
with pytest.raises(HTTPException) as captured:
router.fork_version_for_edit(
"campaign-1",
"version-1",
CampaignVersionUpdateRequest(),
session=object(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
)
assert captured.value.status_code == 403
fork.assert_not_called()

27
tests/test_partial_validation.py
View File

@@ -80,6 +80,30 @@ class CampaignPartialValidationTests(unittest.TestCase):
class CampaignSemanticValidationTests(unittest.TestCase):
def test_send_mode_requires_campaign_owned_sender_for_each_inline_entry(self) -> None:
config = CampaignConfig.model_validate({
"version": "1.0",
"campaign": {"id": "campaign-1", "name": "Campaign", "mode": "send"},
"server": {
"mail_profile_id": "profile-1",
"profile_capabilities": {"smtp_available": True},
},
"recipients": {"allow_individual_from": True},
"template": {"subject": "Subject", "text": "Body", "body_mode": "text"},
"entries": {
"inline": [
{"id": "ready", "from": [{"email": "sender@example.local"}]},
{"id": "missing"},
]
},
})
report = validate_campaign_config(config)
missing_sender = [issue for issue in report.issues if issue.code == "missing_sender"]
self.assertEqual(1, len(missing_sender))
self.assertEqual("/entries/inline/1/from", missing_sender[0].path)
def test_semantic_validation_reports_zip_delivery_and_external_mapping_issues(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -139,8 +163,7 @@ class CampaignSemanticValidationTests(unittest.TestCase):
self.assertIn("unknown_global_value", codes)
self.assertIn("zip_global_password_value_missing", codes)
self.assertIn("zip_archive_unknown", codes)
self.assertIn("delivery_imap_enabled_without_server_imap", codes)
self.assertIn("missing_smtp_config", codes)
self.assertIn("missing_mail_profile", codes)
self.assertIn("unknown_mapping_target", codes)
self.assertIn("mapping_target_not_overridable", codes)
self.assertIn("mapping_columns_missing", codes)

169
tests/test_report_email_security.py Normal file
View File

@@ -0,0 +1,169 @@
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from fastapi import HTTPException
from pydantic import ValidationError
from govoplan_campaign.backend import router
from govoplan_campaign.backend.reports.emailing import CampaignReportEmailError, send_campaign_report_email
from govoplan_campaign.backend.schemas import ReportEmailRequest
def test_report_email_recipient_schema_normalizes_and_has_safe_attachment_default() -> None:
request = ReportEmailRequest.model_validate({
"to": [" First@Example.test ", "first@example.test", "second@example.test"],
})
assert request.to == ["First@Example.test", "second@example.test"]
assert request.attach_jobs_csv is False
@pytest.mark.parametrize(
"recipients",
[
[],
[f"recipient-{index}@example.test" for index in range(51)],
["a" * 310 + "@example.test"],
["victim@example.test\r\nBcc: attacker@example.test"],
["missing-at.example.test"],
["@example.test"],
["local@"],
["local @example.test"],
["one@two@example.test"],
["victim@example.test,Bcc:attacker"],
["Display<a@example.test>"],
],
)
def test_report_email_recipient_schema_rejects_unsafe_addresses(recipients: list[str]) -> None:
with pytest.raises(ValidationError):
ReportEmailRequest.model_validate({"to": recipients})
def test_report_jobs_csv_attachment_requires_recipient_export_permission() -> None:
payload = ReportEmailRequest(to=["recipient@example.test"], attach_jobs_csv=True)
principal = SimpleNamespace(tenant_id="tenant-1")
with (
patch.object(
router,
"_get_campaign_for_principal",
return_value=SimpleNamespace(id="campaign-1", current_version_id=None),
),
patch.object(
router,
"_require_permission",
side_effect=HTTPException(status_code=403, detail="missing export"),
) as require_permission,
patch.object(router, "send_campaign_report_email") as send_report,
):
with pytest.raises(HTTPException) as captured:
router.email_campaign_report(
"campaign-1",
payload,
session=object(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
)
assert captured.value.status_code == 403
require_permission.assert_called_once_with(principal, "campaigns:recipient:export")
send_report.assert_not_called()
def test_report_email_requires_permission_to_use_selected_mail_profile() -> None:
payload = ReportEmailRequest(to=["recipient@example.test"])
principal = SimpleNamespace(tenant_id="tenant-1")
campaign = SimpleNamespace(id="campaign-1", current_version_id="version-1")
version = SimpleNamespace(
id="version-1",
campaign_id="campaign-1",
raw_json={"server": {"mail_profile_id": "profile-1"}},
)
session = SimpleNamespace(get=lambda _model, _id: version)
with (
patch.object(router, "_get_campaign_for_principal", return_value=campaign),
patch.object(
router,
"_require_mail_profile_use_if_needed",
side_effect=HTTPException(status_code=403, detail="missing profile use"),
) as require_profile_use,
patch.object(router, "send_campaign_report_email") as send_report,
):
with pytest.raises(HTTPException) as captured:
router.email_campaign_report(
"campaign-1",
payload,
session=session, # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
)
assert captured.value.status_code == 403
require_profile_use.assert_called_once_with(principal, version.raw_json)
send_report.assert_not_called()
def test_unexpected_report_failure_does_not_leak_internal_details() -> None:
payload = ReportEmailRequest(to=["recipient@example.test"])
principal = SimpleNamespace(tenant_id="tenant-1")
with (
patch.object(
router,
"_get_campaign_for_principal",
return_value=SimpleNamespace(id="campaign-1", current_version_id=None),
),
patch.object(
router,
"send_campaign_report_email",
side_effect=RuntimeError("smtp.internal.example provider-secret"),
),
):
with pytest.raises(HTTPException) as captured:
router.email_campaign_report(
"campaign-1",
payload,
session=object(), # type: ignore[arg-type]
principal=principal, # type: ignore[arg-type]
)
assert captured.value.status_code == 500
assert captured.value.detail == "Campaign report email could not be completed."
def test_report_send_fails_closed_until_durable_mail_outbox_exists() -> None:
campaign = SimpleNamespace(
id="campaign-1",
tenant_id="tenant-1",
name="Campaign",
external_id="campaign-1",
)
version = SimpleNamespace(
id="version-1",
campaign_id="campaign-1",
raw_json={"server": {"mail_profile_id": "profile-1"}},
execution_snapshot={"snapshot_version": "5"},
)
config = SimpleNamespace(
server=SimpleNamespace(profile_capabilities=SimpleNamespace(smtp_available=True)),
)
snapshot = SimpleNamespace(smtp_transport_revision="frozen-build-revision")
class Session:
def get(self, _model, _id):
return campaign
with (
patch("govoplan_campaign.backend.reports.emailing._selected_version", return_value=version),
patch("govoplan_campaign.backend.reports.emailing._load_config", return_value=config),
patch("govoplan_campaign.backend.reports.emailing.ensure_execution_snapshot", return_value=snapshot),
patch("govoplan_campaign.backend.reports.emailing.generate_campaign_report") as generate_report,
):
with pytest.raises(CampaignReportEmailError, match="durable, idempotent Mail-owned outbox"):
send_campaign_report_email(
Session(), # type: ignore[arg-type]
tenant_id="tenant-1",
campaign_id="campaign-1",
to=["recipient@example.test"],
)
generate_report.assert_not_called()

31
tests/test_response_security.py
View File

@@ -40,7 +40,7 @@ def _job() -> SimpleNamespace:
eml_storage_key="campaign/job-1.eml",
claim_token="job-claim-secret",
attempt_count=1,
last_error=None,
last_error="smtp.internal.example /srv/private provider-secret",
queued_at=_now(),
claimed_at=_now(),
smtp_started_at=_now(),
@@ -67,9 +67,9 @@ def _smtp_attempt() -> SimpleNamespace:
status="started",
claim_token="attempt-claim-secret",
smtp_status_code=None,
smtp_response=None,
error_type=None,
error_message=None,
smtp_response="smtp.internal.example provider-secret",
error_type="/srv/private/ProviderError",
error_message="credential=provider-secret",
started_at=_now(),
finished_at=None,
)
@@ -82,7 +82,7 @@ def _imap_attempt() -> SimpleNamespace:
status="claimed",
claim_token="imap-claim-secret",
folder="Sent",
error_message=None,
error_message="imap.internal.example /srv/private provider-secret",
created_at=_now(),
updated_at=_now(),
)
@@ -148,6 +148,7 @@ def test_version_response_omits_source_base_path_and_sanitizes_summaries() -> No
"campaign": {"title": "Public"},
"files": [{"storage_key": "private/object", "filename": "public.pdf"}],
"server": {
"mail_profile_id": "profile-1",
"smtp": {"host": "smtp.example.invalid", "password": "smtp-secret"},
"imap": {"host": "imap.example.invalid", "password": "imap-secret"},
"credentials": {
@@ -176,14 +177,7 @@ def test_version_response_omits_source_base_path_and_sanitizes_summaries() -> No
assert detail["raw_json"] == {
"campaign": {"title": "Public"},
"files": [{"filename": "public.pdf"}],
"server": {
"smtp": {"host": "smtp.example.invalid"},
"imap": {"host": "imap.example.invalid"},
"credentials": {
"smtp": {"username": "sender"},
"imap": {"username": "archive"},
},
},
"server": {"mail_profile_id": "profile-1"},
"archives": [{"password": "business-zip-password"}],
"template": {
"source": {
@@ -209,6 +203,14 @@ def test_ordinary_job_detail_and_attempts_do_not_expose_diagnostics() -> None:
assert job_payload["attachments"] == [{"filename": "public.pdf"}]
assert "claim_token" not in attempts["smtp"][0]
assert "claim_token" not in attempts["imap"][0]
assert "smtp_response" not in attempts["smtp"][0]
assert "error_type" not in attempts["smtp"][0]
assert "error_message" not in attempts["smtp"][0]
assert "error_message" not in attempts["imap"][0]
ordinary = repr({"job": job_payload, "attempts": attempts})
assert "internal.example" not in ordinary
assert "/srv/private" not in ordinary
assert "provider-secret" not in ordinary
def test_operator_diagnostics_include_claim_and_storage_details() -> None:
@@ -223,6 +225,9 @@ def test_operator_diagnostics_include_claim_and_storage_details() -> None:
assert diagnostics["worker_claim"]["claim_token"] == "job-claim-secret"
assert diagnostics["attempts"]["smtp"][0]["claim_token"] == "attempt-claim-secret"
assert diagnostics["attempts"]["imap"][0]["claim_token"] == "imap-claim-secret"
assert "smtp.internal.example" in diagnostics["attempts"]["smtp"][0]["smtp_response"]
assert "imap.internal.example" in diagnostics["attempts"]["imap"][0]["error_message"]
assert "provider-secret" in diagnostics["worker_claim"]["last_error"]
def test_diagnostics_permission_is_operator_only_by_default() -> None:

75
tests/test_sending_jobs.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from govoplan_campaign.backend.db.models import (
JobBuildStatus,
@@ -10,8 +11,10 @@ from govoplan_campaign.backend.db.models import (
JobValidationStatus,
)
from govoplan_campaign.backend.sending.jobs import (
SendJobResult,
_queue_validation_statuses,
_select_campaign_jobs_for_queue,
_send_claimed_campaign_job,
)
@@ -115,6 +118,78 @@ class CampaignQueueSelectionTests(unittest.TestCase):
self.assertEqual(warning.send_status, JobSendStatus.NOT_QUEUED.value)
self.assertEqual(session.added, [])
def test_post_smtp_persistence_failure_is_outcome_unknown_not_retryable_failure(self):
job = SimpleNamespace(
id="job-1",
tenant_id="tenant-1",
campaign_id="campaign-1",
campaign_version_id="version-1",
imap_status="not_requested",
resolved_recipients={"from": {"email": "sender@example.test"}},
)
snapshot = SimpleNamespace(
mail_profile_id="profile-1",
smtp_transport_revision="frozen",
delivery=SimpleNamespace(
rate_limit=SimpleNamespace(messages_per_minute=60),
),
)
context = SimpleNamespace(
snapshot=snapshot,
message_bytes=b"message",
envelope_from="sender@example.test",
envelope_recipients=["recipient@example.test"],
)
current = SimpleNamespace(id="job-1")
class Session:
def __init__(self) -> None:
self.rolled_back = False
def rollback(self) -> None:
self.rolled_back = True
def get(self, _model, _id):
return current
class Mail:
def wait_for_rate_limit(self, **_kwargs):
return None
def send_campaign_email_bytes(self, *_args, **_kwargs):
return SimpleNamespace(accepted_count=1)
session = Session()
expected = SendJobResult(
job_id="job-1",
status=JobSendStatus.OUTCOME_UNKNOWN.value,
attempt_number=1,
)
with (
patch("govoplan_campaign.backend.sending.jobs.mail_integration", return_value=Mail()),
patch("govoplan_campaign.backend.sending.jobs._record_attempt_start", return_value=object()),
patch(
"govoplan_campaign.backend.sending.jobs._record_smtp_send_success",
side_effect=OSError("storage unavailable"),
),
patch(
"govoplan_campaign.backend.sending.jobs.mark_job_outcome_unknown",
return_value=expected,
) as mark_unknown,
):
result = _send_claimed_campaign_job(
session, # type: ignore[arg-type]
job=job, # type: ignore[arg-type]
claim_token="claim-1",
context=context, # type: ignore[arg-type]
use_rate_limit=False,
enqueue_imap_task=False,
)
self.assertIs(result, expected)
self.assertTrue(session.rolled_back)
self.assertIn("Automatic retry is stopped", mark_unknown.call_args.kwargs["reason"])
if __name__ == "__main__":
unittest.main()