intermittent commit

This commit is contained in:
2026-07-14 13:22:10 +02:00
parent 3f0b14a726
commit 2e593b7fa4
31 changed files with 3523 additions and 1672 deletions

View File

@@ -6,7 +6,14 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from govoplan_core.mail.config import ImapConfig, ImapServerConfig, SmtpConfig, SmtpServerConfig, TransportCredentials, TransportSecurity
from govoplan_core.mail.config import (
ImapConfig,
ImapServerConfig,
SmtpConfig,
SmtpServerConfig,
TransportCredentials,
normalize_split_transport_credentials,
)
class StrictModel(BaseModel):
@@ -124,28 +131,7 @@ class ServerConfig(StrictModel):
@model_validator(mode="before")
@classmethod
def normalize_legacy_credentials(cls, value: Any) -> Any:
if not isinstance(value, dict):
return value
data = dict(value)
credentials = data.get("credentials") if isinstance(data.get("credentials"), dict) else {}
credentials = {key: dict(item) for key, item in credentials.items() if isinstance(item, dict)}
for protocol in ("smtp", "imap"):
transport = data.get(protocol)
if not isinstance(transport, dict):
continue
next_transport = dict(transport)
next_credentials = dict(credentials.get(protocol) or {})
for field in ("username", "password"):
if field in next_transport and field not in next_credentials:
next_credentials[field] = next_transport[field]
next_transport.pop(field, None)
next_transport.pop("enabled", None)
data[protocol] = next_transport
if next_credentials:
credentials[protocol] = next_credentials
if credentials:
data["credentials"] = credentials
return data
return normalize_split_transport_credentials(value)
def runtime_smtp_config(self) -> SmtpConfig | None:
if self.smtp is None:
@@ -501,7 +487,11 @@ class ImportProvenance(StrictModel):
id: str
imported_at: str
mode: Literal["append", "replace"]
source_type: Literal["csv", "xlsx", "text"]
source_type: Literal["csv", "xlsx", "text", "addresses"]
source_id: str | None = None
source_label: str | None = None
source_revision: str | None = None
source_provenance: dict[str, Any] = Field(default_factory=dict)
filename: str | None = None
sheet_name: str | None = None
encoding: str | None = None

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import csv
from dataclasses import dataclass
from enum import StrEnum
from pathlib import Path
from typing import Iterable
@@ -8,7 +9,7 @@ from typing import Iterable
from pydantic import BaseModel, ConfigDict, Field
from .field_values import ignored_entry_field_overrides
from .models import AttachmentConfig, CampaignConfig, EntryConfig, FieldType, SourceType, ZipPasswordMode, ZipPasswordScope, ZipRuleMode
from .models import AttachmentConfig, CampaignConfig, EntryConfig, FieldType, SourceType, ZipArchiveConfig, ZipPasswordMode, ZipPasswordScope, ZipRuleMode
class Severity(StrEnum):
@@ -51,6 +52,13 @@ class SemanticReport(BaseModel):
return self.error_count == 0
@dataclass(frozen=True, slots=True)
class _EntriesValidation:
mode: str
count: int | None
issues: list[SemanticIssue]
def _issue(severity: Severity, code: str, message: str, path: str | None = None) -> SemanticIssue:
return SemanticIssue(severity=severity, code=code, message=message, path=path)
@@ -200,57 +208,95 @@ def _attachment_path_issues(config: CampaignConfig) -> list[SemanticIssue]:
def _zip_configuration_issues(config: CampaignConfig) -> list[SemanticIssue]:
collection = config.attachments.zip
issues: list[SemanticIssue] = []
if not collection.enabled:
return issues
return []
if not collection.archives:
return [_issue(Severity.ERROR, "zip_archive_missing", "Attachment zipping is enabled, but no ZIP archive is configured", "/attachments/zip/archives")]
issues, archive_ids, standard_count = _zip_archive_catalog_issues(config)
if standard_count != 1:
issues.append(_issue(Severity.ERROR, "zip_standard_archive_invalid", "Exactly one ZIP archive must be selected as the campaign standard", "/attachments/zip/archives"))
issues.extend(_zip_rule_selection_issues(config, archive_ids))
return issues
def _zip_archive_catalog_issues(config: CampaignConfig) -> tuple[list[SemanticIssue], set[str], int]:
issues: list[SemanticIssue] = []
archive_ids: set[str] = set()
archive_names: set[str] = set()
standard_count = 0
field_definitions = {field.name: field for field in config.fields}
for index, archive in enumerate(collection.archives):
for index, archive in enumerate(config.attachments.zip.archives):
path = f"/attachments/zip/archives/{index}"
if not archive.id.strip():
issues.append(_issue(Severity.ERROR, "zip_archive_id_missing", "ZIP archive has no identifier", f"{path}/id"))
elif archive.id in archive_ids:
issues.append(_issue(Severity.ERROR, "zip_archive_id_duplicate", f"ZIP archive id {archive.id!r} is used more than once", f"{path}/id"))
archive_ids.add(archive.id)
normalized_archive_name = _normalized_zip_archive_name(archive.name)
if not normalized_archive_name:
issues.append(_issue(Severity.ERROR, "zip_archive_name_missing", "ZIP archive has no filename", f"{path}/name"))
elif normalized_archive_name in archive_names:
issues.append(_issue(Severity.ERROR, "zip_archive_name_duplicate", f"ZIP archive filename {archive.name!r} is used more than once; archive filenames must be unique", f"{path}/name"))
archive_names.add(normalized_archive_name)
issues.extend(_zip_archive_identity_issues(archive, path, archive_ids, archive_names))
if archive.standard:
standard_count += 1
issues.extend(_zip_password_issues(config, archive, path, field_definitions))
return issues, archive_ids, standard_count
if not archive.password_enabled:
continue
if archive.password_mode == ZipPasswordMode.DIRECT:
if not (archive.password or ""):
issues.append(_issue(Severity.ERROR, "zip_password_missing", "A legacy fixed ZIP password is enabled, but no password is configured", f"{path}/password"))
continue
if archive.password_mode == ZipPasswordMode.TEMPLATE:
if not (archive.password_template or ""):
issues.append(_issue(Severity.ERROR, "zip_password_template_missing", "A legacy ZIP password template is enabled, but no template is configured", f"{path}/password_template"))
continue
field_name = (archive.password_field or "").strip()
field = field_definitions.get(field_name)
if not field_name:
issues.append(_issue(Severity.ERROR, "zip_password_field_missing", f"ZIP archive {archive.name!r} has password protection enabled, but no field is selected", f"{path}/password_field"))
elif field is None:
issues.append(_issue(Severity.ERROR, "zip_password_field_unknown", f"ZIP password field {field_name!r} is not declared in campaign fields", f"{path}/password_field"))
elif field.type != FieldType.PASSWORD:
issues.append(_issue(Severity.WARNING, "zip_password_field_not_password_type", f"ZIP password field {field_name!r} is not configured with field type 'password'", f"{path}/password_field"))
elif archive.password_scope == ZipPasswordScope.GLOBAL and config.global_values.get(field_name) in (None, ""):
issues.append(_issue(Severity.ERROR, "zip_global_password_value_missing", f"Global ZIP password field {field_name!r} has no campaign-wide value", f"/global_values/{field_name}"))
def _zip_archive_identity_issues(
archive: ZipArchiveConfig,
path: str,
archive_ids: set[str],
archive_names: set[str],
) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
if not archive.id.strip():
issues.append(_issue(Severity.ERROR, "zip_archive_id_missing", "ZIP archive has no identifier", f"{path}/id"))
elif archive.id in archive_ids:
issues.append(_issue(Severity.ERROR, "zip_archive_id_duplicate", f"ZIP archive id {archive.id!r} is used more than once", f"{path}/id"))
archive_ids.add(archive.id)
if standard_count != 1:
issues.append(_issue(Severity.ERROR, "zip_standard_archive_invalid", "Exactly one ZIP archive must be selected as the campaign standard", "/attachments/zip/archives"))
normalized_archive_name = _normalized_zip_archive_name(archive.name)
if not normalized_archive_name:
issues.append(_issue(Severity.ERROR, "zip_archive_name_missing", "ZIP archive has no filename", f"{path}/name"))
elif normalized_archive_name in archive_names:
issues.append(_issue(Severity.ERROR, "zip_archive_name_duplicate", f"ZIP archive filename {archive.name!r} is used more than once; archive filenames must be unique", f"{path}/name"))
archive_names.add(normalized_archive_name)
return issues
def _zip_password_issues(
config: CampaignConfig,
archive: ZipArchiveConfig,
path: str,
field_definitions: dict[str, object],
) -> list[SemanticIssue]:
if not archive.password_enabled:
return []
if archive.password_mode == ZipPasswordMode.DIRECT:
if not (archive.password or ""):
return [_issue(Severity.ERROR, "zip_password_missing", "A legacy fixed ZIP password is enabled, but no password is configured", f"{path}/password")]
return []
if archive.password_mode == ZipPasswordMode.TEMPLATE:
if not (archive.password_template or ""):
return [_issue(Severity.ERROR, "zip_password_template_missing", "A legacy ZIP password template is enabled, but no template is configured", f"{path}/password_template")]
return []
return _zip_password_field_issues(config, archive, path, field_definitions)
def _zip_password_field_issues(
config: CampaignConfig,
archive: ZipArchiveConfig,
path: str,
field_definitions: dict[str, object],
) -> list[SemanticIssue]:
field_name = (archive.password_field or "").strip()
field = field_definitions.get(field_name)
if not field_name:
return [_issue(Severity.ERROR, "zip_password_field_missing", f"ZIP archive {archive.name!r} has password protection enabled, but no field is selected", f"{path}/password_field")]
if field is None:
return [_issue(Severity.ERROR, "zip_password_field_unknown", f"ZIP password field {field_name!r} is not declared in campaign fields", f"{path}/password_field")]
if getattr(field, "type", None) != FieldType.PASSWORD:
return [_issue(Severity.WARNING, "zip_password_field_not_password_type", f"ZIP password field {field_name!r} is not configured with field type 'password'", f"{path}/password_field")]
if archive.password_scope == ZipPasswordScope.GLOBAL and config.global_values.get(field_name) in (None, ""):
return [_issue(Severity.ERROR, "zip_global_password_value_missing", f"Global ZIP password field {field_name!r} has no campaign-wide value", f"/global_values/{field_name}")]
return []
def _zip_rule_selection_issues(config: CampaignConfig, archive_ids: set[str]) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
for path, rule, _is_individual in _iter_attachment_rules(config):
selection = (rule.zip.archive_id or ZipRuleMode.INHERIT.value).strip()
if selection not in {"", ZipRuleMode.INHERIT.value, ZipRuleMode.INCLUDE.value, ZipRuleMode.EXCLUDE.value} and selection not in archive_ids:
@@ -276,6 +322,236 @@ def _ignored_override_issues(config: CampaignConfig, entry: EntryConfig, path_pr
]
def _global_value_issues(config: CampaignConfig, declared_names: set[str]) -> list[SemanticIssue]:
return [
_issue(
Severity.WARNING,
"unknown_global_value",
f"global_values contains {key!r}, but it is not declared in fields",
f"/global_values/{key}",
)
for key in config.global_values
if declared_names and key not in declared_names
]
def _delivery_issues(config: CampaignConfig) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
runtime_imap = config.server.runtime_imap_config()
if config.delivery.imap_append_sent.enabled:
issues.extend(_imap_delivery_issues(runtime_imap))
runtime_smtp = config.server.runtime_smtp_config()
if config.campaign.mode == "send" and not runtime_smtp:
issues.append(
_issue(
Severity.ERROR,
"missing_smtp_config",
"campaign mode is 'send', but no server.smtp configuration is present",
"/server/smtp",
)
)
if runtime_smtp:
missing = [name for name in ["host", "port"] if getattr(runtime_smtp, name) in (None, "")]
if missing:
issues.append(
_issue(
Severity.WARNING,
"incomplete_smtp_config",
"SMTP settings are present, but these settings are missing: " + ", ".join(missing),
"/server/smtp",
)
)
return issues
def _imap_delivery_issues(runtime_imap: object | None) -> list[SemanticIssue]:
if runtime_imap is None:
return [
_issue(
Severity.WARNING,
"delivery_imap_enabled_without_server_imap",
"delivery.imap_append_sent is enabled, but no server.imap configuration is present",
"/delivery/imap_append_sent/enabled",
)
]
missing = [name for name in ["host", "port", "username", "password"] if getattr(runtime_imap, name) in (None, "")]
if not missing:
return []
return [
_issue(
Severity.ERROR,
"incomplete_imap_config",
"IMAP append is enabled, but these IMAP settings are missing: " + ", ".join(missing),
"/server/imap",
)
]
def _entries_validation(
config: CampaignConfig,
*,
campaign_path: Path,
check_files: bool,
field_names: set[str],
field_definitions: dict[str, object],
) -> _EntriesValidation:
if config.entries.is_inline:
inline_entries = config.entries.inline or []
issues = _inline_entries_issues(config, inline_entries)
return _EntriesValidation(mode="inline", count=len(inline_entries), issues=issues)
issues = _external_entries_issues(
config,
campaign_path=campaign_path,
check_files=check_files,
field_names=field_names,
field_definitions=field_definitions,
)
source_type = config.entries.source.type.value if config.entries.source else "unknown"
return _EntriesValidation(mode=f"external:{source_type}", count=None, issues=issues)
def _inline_entries_issues(config: CampaignConfig, inline_entries: list[EntryConfig]) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
if not inline_entries:
issues.append(_issue(Severity.WARNING, "no_inline_entries", "entries.inline is empty", "/entries/inline"))
for index, entry in enumerate(inline_entries):
if entry.active:
issues.extend(_ignored_override_issues(config, entry, f"/entries/inline/{index}"))
return issues
def _external_entries_issues(
config: CampaignConfig,
*,
campaign_path: Path,
check_files: bool,
field_names: set[str],
field_definitions: dict[str, object],
) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
mapping = config.entries.mapping or {}
if not mapping:
issues.append(_issue(Severity.ERROR, "empty_mapping", "external entries require a non-empty mapping", "/entries/mapping"))
issues.extend(_mapping_issues(mapping, field_names, field_definitions))
if config.entries.defaults:
issues.extend(_ignored_override_issues(config, config.entries.defaults, "/entries/defaults"))
if check_files and config.entries.source:
issues.extend(_entries_source_file_issues(config, campaign_path, mapping))
return issues
def _mapping_issues(
mapping: dict[str, str],
field_names: set[str],
field_definitions: dict[str, object],
) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
for target in mapping:
if not _mapping_target_known(target, field_names):
issues.append(
_issue(
Severity.WARNING,
"unknown_mapping_target",
f"mapping target {target!r} is not recognized by the current campaign model",
f"/entries/mapping/{target}",
)
)
field_name = _mapping_target_field_name(target)
field = field_definitions.get(field_name or "")
if field_name and field is not None and not getattr(field, "can_override", True):
issues.append(
_issue(
Severity.WARNING,
"mapping_target_not_overridable",
f"mapping target {target!r} points to a field that does not allow recipient overrides; mapped values will be ignored",
f"/entries/mapping/{target}",
)
)
return issues
def _entries_source_file_issues(config: CampaignConfig, campaign_path: Path, mapping: dict[str, str]) -> list[SemanticIssue]:
source = config.entries.source
if source is None:
return []
source_path = _resolve(campaign_path, source.path)
if not source_path.exists():
return [
_issue(
Severity.ERROR,
"entries_source_not_found",
f"entries source file does not exist: {source_path}",
"/entries/source/path",
)
]
if source.type != SourceType.CSV or not source.has_header:
return []
try:
header = _csv_header(source_path, source.delimiter, source.encoding)
except OSError as exc:
return [_issue(Severity.ERROR, "entries_source_read_error", str(exc), "/entries/source/path")]
header_set = set(header or [])
missing_columns = sorted({source_name for source_name in mapping.values() if source_name not in header_set})
if not missing_columns:
return []
return [
_issue(
Severity.ERROR,
"mapping_columns_missing",
"CSV mapping refers to missing columns: " + ", ".join(missing_columns),
"/entries/mapping",
)
]
def _file_check_issues(config: CampaignConfig, campaign_path: Path) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
issues.extend(_attachment_file_check_issues(config, campaign_path))
issues.extend(_template_source_file_issues(config, campaign_path))
return issues
def _attachment_file_check_issues(config: CampaignConfig, campaign_path: Path) -> list[SemanticIssue]:
if config.attachments.base_paths:
return [
_issue(
Severity.WARNING,
"attachments_base_path_not_found",
f"attachment base path {base_path_config.name!r} does not exist: {_resolve(campaign_path, base_path_config.path)}",
f"/attachments/base_paths/{index}/path",
)
for index, base_path_config in enumerate(config.attachments.base_paths)
if not _resolve(campaign_path, base_path_config.path).exists()
]
attachments_base_path = _resolve(campaign_path, config.attachments.base_path)
if attachments_base_path.exists():
return []
return [
_issue(
Severity.WARNING,
"attachments_base_path_not_found",
f"attachments.base_path does not exist: {attachments_base_path}",
"/attachments/base_path",
)
]
def _template_source_file_issues(config: CampaignConfig, campaign_path: Path) -> list[SemanticIssue]:
issues: list[SemanticIssue] = []
for schema_path, raw_path in _iter_template_source_paths(config):
path = _resolve(campaign_path, raw_path)
if not path.exists():
issues.append(
_issue(
Severity.ERROR,
"template_source_not_found",
f"template source file does not exist: {path}",
schema_path,
)
)
return issues
def validate_campaign_config(
config: CampaignConfig,
*,
@@ -289,149 +565,29 @@ def validate_campaign_config(
field_definitions = {field.name: field for field in config.fields}
declared_names = set(field_definitions)
for key in config.global_values:
if declared_names and key not in declared_names:
issues.append(_issue(
Severity.WARNING,
"unknown_global_value",
f"global_values contains {key!r}, but it is not declared in fields",
f"/global_values/{key}",
))
issues.extend(_global_value_issues(config, declared_names))
issues.extend(_attachment_path_issues(config))
issues.extend(_zip_configuration_issues(config))
issues.extend(_delivery_issues(config))
runtime_imap = config.server.runtime_imap_config()
if config.delivery.imap_append_sent.enabled:
if runtime_imap is None:
issues.append(_issue(
Severity.WARNING,
"delivery_imap_enabled_without_server_imap",
"delivery.imap_append_sent is enabled, but no server.imap configuration is present",
"/delivery/imap_append_sent/enabled",
))
else:
missing = [name for name in ["host", "port", "username", "password"] if getattr(runtime_imap, name) in (None, "")]
if missing:
issues.append(_issue(
Severity.ERROR,
"incomplete_imap_config",
"IMAP append is enabled, but these IMAP settings are missing: " + ", ".join(missing),
"/server/imap",
))
runtime_smtp = config.server.runtime_smtp_config()
if config.campaign.mode == "send" and not runtime_smtp:
issues.append(_issue(
Severity.ERROR,
"missing_smtp_config",
"campaign mode is 'send', but no server.smtp configuration is present",
"/server/smtp",
))
if runtime_smtp:
missing = [name for name in ["host", "port"] if getattr(runtime_smtp, name) in (None, "")]
if missing:
issues.append(_issue(
Severity.WARNING,
"incomplete_smtp_config",
"SMTP settings are present, but these settings are missing: " + ", ".join(missing),
"/server/smtp",
))
if config.entries.is_inline:
inline_entries = config.entries.inline or []
entries_count = len(inline_entries)
entries_mode = "inline"
if entries_count == 0:
issues.append(_issue(Severity.WARNING, "no_inline_entries", "entries.inline is empty", "/entries/inline"))
for index, entry in enumerate(inline_entries):
if entry.active:
issues.extend(_ignored_override_issues(config, entry, f"/entries/inline/{index}"))
else:
entries_count = None
entries_mode = f"external:{config.entries.source.type.value if config.entries.source else 'unknown'}"
mapping = config.entries.mapping or {}
if not mapping:
issues.append(_issue(Severity.ERROR, "empty_mapping", "external entries require a non-empty mapping", "/entries/mapping"))
for target in mapping:
if not _mapping_target_known(target, field_names):
issues.append(_issue(
Severity.WARNING,
"unknown_mapping_target",
f"mapping target {target!r} is not recognized by the current campaign model",
f"/entries/mapping/{target}",
))
field_name = _mapping_target_field_name(target)
if field_name and field_name in field_definitions and not field_definitions[field_name].can_override:
issues.append(_issue(
Severity.WARNING,
"mapping_target_not_overridable",
f"mapping target {target!r} points to a field that does not allow recipient overrides; mapped values will be ignored",
f"/entries/mapping/{target}",
))
if config.entries.defaults:
issues.extend(_ignored_override_issues(config, config.entries.defaults, "/entries/defaults"))
if check_files and config.entries.source:
source_path = _resolve(campaign_path, config.entries.source.path)
if not source_path.exists():
issues.append(_issue(
Severity.ERROR,
"entries_source_not_found",
f"entries source file does not exist: {source_path}",
"/entries/source/path",
))
elif config.entries.source.type == SourceType.CSV and config.entries.source.has_header:
try:
header = _csv_header(source_path, config.entries.source.delimiter, config.entries.source.encoding)
header_set = set(header or [])
missing_columns = sorted({source_name for source_name in mapping.values() if source_name not in header_set})
if missing_columns:
issues.append(_issue(
Severity.ERROR,
"mapping_columns_missing",
"CSV mapping refers to missing columns: " + ", ".join(missing_columns),
"/entries/mapping",
))
except OSError as exc:
issues.append(_issue(Severity.ERROR, "entries_source_read_error", str(exc), "/entries/source/path"))
entries = _entries_validation(
config,
campaign_path=campaign_path,
check_files=check_files,
field_names=field_names,
field_definitions=field_definitions,
)
issues.extend(entries.issues)
if check_files:
if config.attachments.base_paths:
for index, base_path_config in enumerate(config.attachments.base_paths):
attachments_base_path = _resolve(campaign_path, base_path_config.path)
if not attachments_base_path.exists():
issues.append(_issue(
Severity.WARNING,
"attachments_base_path_not_found",
f"attachment base path {base_path_config.name!r} does not exist: {attachments_base_path}",
f"/attachments/base_paths/{index}/path",
))
else:
attachments_base_path = _resolve(campaign_path, config.attachments.base_path)
if not attachments_base_path.exists():
issues.append(_issue(
Severity.WARNING,
"attachments_base_path_not_found",
f"attachments.base_path does not exist: {attachments_base_path}",
"/attachments/base_path",
))
for schema_path, raw_path in _iter_template_source_paths(config):
path = _resolve(campaign_path, raw_path)
if not path.exists():
issues.append(_issue(
Severity.ERROR,
"template_source_not_found",
f"template source file does not exist: {path}",
schema_path,
))
issues.extend(_file_check_issues(config, campaign_path))
report = SemanticReport(
campaign_id=config.campaign.id,
campaign_name=config.campaign.name,
issues=issues,
entries_mode=entries_mode,
entries_count=entries_count,
entries_mode=entries.mode,
entries_count=entries.count,
attachments_base_path=_attachment_base_path_report_value(config),
rate_limit=f"{config.delivery.rate_limit.messages_per_minute}/min, concurrency {config.delivery.rate_limit.concurrency}",
imap_append_enabled=config.delivery.imap_append_sent.enabled,

View File

@@ -3,10 +3,17 @@ from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy import event, inspect
from sqlalchemy import event
from sqlalchemy.orm import Session as OrmSession
from govoplan_core.core.change_sequence import record_change
from govoplan_core.core.sqlalchemy_change_tracking import (
ensure_object_id,
has_attr_changes,
object_state,
operation_for_object,
previous_value,
)
from govoplan_campaign.backend.db.models import (
Campaign,
CampaignIssue,
@@ -85,10 +92,10 @@ def _record_campaign_change(session: OrmSession, campaign: Campaign) -> None:
def _record_share_visibility_change(session: OrmSession, share: CampaignShare) -> None:
state = inspect(share)
state = object_state(share)
if not share.campaign_id:
return
if not state.deleted and not state.pending and not _has_attr_changes(
if not state.deleted and not state.pending and not has_attr_changes(
state,
("campaign_id", "target_type", "target_id", "permission", "revoked_at"),
):
@@ -271,12 +278,12 @@ def _record_attempt_change(session: OrmSession, attempt: SendAttempt | ImapAppen
def _operation_for_campaign(obj: Campaign, *, changed_attrs: tuple[str, ...]) -> str | None:
state = inspect(obj)
state = object_state(obj)
if state.pending:
return "created"
if state.deleted:
return "deleted"
if not _has_attr_changes(state, changed_attrs):
if not has_attr_changes(state, changed_attrs):
return None
status_history = state.attrs.status.history
if status_history.has_changes() and any(value == "deleted" for value in status_history.added):
@@ -285,38 +292,11 @@ def _operation_for_campaign(obj: Campaign, *, changed_attrs: tuple[str, ...]) ->
def _operation_for_object(obj: object, *, changed_attrs: tuple[str, ...]) -> str | None:
state = inspect(obj)
if state.pending:
return "created"
if state.deleted:
return "deleted"
if not _has_attr_changes(state, changed_attrs):
return None
return "updated"
def _has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool:
return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs)
def _previous_value(obj: object, attr_name: str) -> str | None:
state = inspect(obj)
if attr_name not in state.attrs:
return None
history = state.attrs[attr_name].history
if not history.has_changes() or not history.deleted:
return None
value = history.deleted[0]
return str(value) if value is not None else None
return operation_for_object(obj, changed_attrs=changed_attrs)
def _ensure_id(obj: object) -> str:
resource_id = getattr(obj, "id", None)
if resource_id:
return str(resource_id)
resource_id = new_uuid()
setattr(obj, "id", resource_id)
return resource_id
return ensure_object_id(obj, new_uuid)
def _campaign_payload(campaign: Campaign | None) -> dict[str, Any]:
@@ -326,12 +306,12 @@ def _campaign_payload(campaign: Campaign | None) -> dict[str, Any]:
"campaign_id": campaign.id,
"owner_user_id": campaign.owner_user_id,
"owner_group_id": campaign.owner_group_id,
"previous_owner_user_id": _previous_value(campaign, "owner_user_id"),
"previous_owner_group_id": _previous_value(campaign, "owner_group_id"),
"previous_owner_user_id": previous_value(campaign, "owner_user_id"),
"previous_owner_group_id": previous_value(campaign, "owner_group_id"),
"status": campaign.status,
"previous_status": _previous_value(campaign, "status"),
"previous_status": previous_value(campaign, "status"),
"current_version_id": campaign.current_version_id,
"previous_current_version_id": _previous_value(campaign, "current_version_id"),
"previous_current_version_id": previous_value(campaign, "current_version_id"),
}

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
from dataclasses import dataclass
from email import policy
from email.message import EmailMessage
from typing import Any
@@ -19,6 +20,30 @@ class MockCampaignSendError(RuntimeError):
pass
@dataclass(slots=True)
class _MockSendOutcome:
row: dict[str, Any]
sent_count: int = 0
failed_count: int = 0
skipped_count: int = 0
imap_appended_count: int = 0
imap_failed_count: int = 0
@dataclass(slots=True)
class _MockSendBatch:
results: list[dict[str, Any]]
sent_count: int = 0
failed_count: int = 0
skipped_count: int = 0
imap_appended_count: int = 0
imap_failed_count: int = 0
@property
def attempted_count(self) -> int:
return self.sent_count + self.failed_count
def _mock_mailbox() -> Any | None:
return mail_integration().mock_mailbox()
@@ -116,6 +141,163 @@ def _can_mock_send(
return False, f"Validation status is {message.validation_status.value}"
def _mock_send_batch(
*,
config: Any,
built_messages: list[Any],
mailbox: Any | None,
send: bool,
include_warnings: bool,
include_needs_review: bool,
append_sent: bool,
) -> _MockSendBatch:
batch = _MockSendBatch(results=[])
for built in built_messages:
outcome = _mock_send_message(
config=config,
built=built,
mailbox=mailbox,
send=send,
include_warnings=include_warnings,
include_needs_review=include_needs_review,
append_sent=append_sent,
)
batch.results.append(outcome.row)
batch.sent_count += outcome.sent_count
batch.failed_count += outcome.failed_count
batch.skipped_count += outcome.skipped_count
batch.imap_appended_count += outcome.imap_appended_count
batch.imap_failed_count += outcome.imap_failed_count
return batch
def _mock_send_message(
*,
config: Any,
built: Any,
mailbox: Any | None,
send: bool,
include_warnings: bool,
include_needs_review: bool,
append_sent: bool,
) -> _MockSendOutcome:
draft = built.draft
row = _mock_send_row(draft)
can_send, skip_reason = _can_mock_send(
draft,
include_warnings=include_warnings,
include_needs_review=include_needs_review,
)
if not can_send or built.mime is None:
row.update({"status": "skipped", "message": skip_reason or "Message has no MIME output"})
return _MockSendOutcome(row=row, skipped_count=1)
recipients = _recipient_emails(draft)
envelope_from = _envelope_from(draft)
if not recipients:
row.update({"status": "skipped", "message": "No envelope recipients"})
return _MockSendOutcome(row=row, skipped_count=1)
if not send:
row.update({
"status": "ready",
"message": f"Would send to {len(recipients)} recipient(s)",
"envelope_from": envelope_from,
"envelope_recipients": recipients,
})
return _MockSendOutcome(row=row)
return _send_mock_smtp_message(
config=config,
built=built,
mailbox=mailbox,
row=row,
recipients=recipients,
envelope_from=envelope_from,
append_sent=append_sent,
)
def _mock_send_row(draft: MessageDraft) -> dict[str, Any]:
return {
"entry_index": draft.entry_index,
"entry_id": draft.entry_id,
"subject": draft.subject,
"validation_status": draft.validation_status.value,
"build_status": str(draft.build_status.value if hasattr(draft.build_status, "value") else draft.build_status),
"to": _message_addresses_payload(draft.to),
"attachments": _attachment_payloads(draft),
"issues": _issue_payloads(draft),
}
def _send_mock_smtp_message(
*,
config: Any,
built: Any,
mailbox: Any | None,
row: dict[str, Any],
recipients: list[str],
envelope_from: str,
append_sent: bool,
) -> _MockSendOutcome:
try:
if mailbox is not None and mailbox.consume_fail_next_smtp():
raise MockCampaignSendError("Configured mock failure: next SMTP delivery fails")
rejected = _smtp_rejection_matches(recipients)
if rejected and len(rejected) == len(recipients):
raise MockCampaignSendError(f"Configured mock failure: all recipients rejected ({', '.join(rejected)})")
accepted = [recipient for recipient in recipients if recipient not in rejected]
smtp_record = mailbox.record_smtp_delivery(
built.mime,
envelope_from=envelope_from,
envelope_recipients=accepted,
smtp_host="mock.smtp.local",
)
row.update({
"status": "sent",
"message": f"Mock SMTP captured as {smtp_record.id}",
"smtp_message_id": smtp_record.id,
"envelope_from": envelope_from,
"envelope_recipients": accepted,
"refused_recipients": rejected,
})
imap_appended, imap_failed = _append_mock_sent_message(config, built, mailbox, row, append_sent=append_sent)
return _MockSendOutcome(row=row, sent_count=1, imap_appended_count=imap_appended, imap_failed_count=imap_failed)
except Exception as exc:
row.update({"status": "failed", "message": str(exc), "envelope_from": envelope_from, "envelope_recipients": recipients})
return _MockSendOutcome(row=row, failed_count=1)
def _append_mock_sent_message(
config: Any,
built: Any,
mailbox: Any | None,
row: dict[str, Any],
*,
append_sent: bool,
) -> tuple[int, int]:
if not append_sent:
return 0, 0
try:
if mailbox is not None and mailbox.consume_fail_next_imap():
raise MockCampaignSendError("Configured mock failure: next IMAP append fails")
folder = _mock_sent_folder(config)
imap_record = mailbox.record_imap_append(_raw_message_bytes(built.mime), folder=folder, imap_host="mock.imap.local")
row.update({"imap_status": "appended", "imap_message_id": imap_record.id, "imap_folder": folder})
return 1, 0
except Exception as exc:
row.update({"imap_status": "failed", "imap_error": str(exc)})
return 0, 1
def _mock_sent_folder(config: Any) -> str:
if config.delivery.imap_append_sent.folder and config.delivery.imap_append_sent.folder != "auto":
return config.delivery.imap_append_sent.folder
if config.server.imap and config.server.imap.sent_folder and config.server.imap.sent_folder != "auto":
return config.server.imap.sent_folder
return "Sent"
def run_mock_campaign_send(
session: Session,
*,
@@ -158,7 +340,7 @@ def run_mock_campaign_send(
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=True,
prefix="multimailer-mock-send-",
prefix="govoplan-mock-send-",
) as prepared:
prepared_raw = load_campaign_json(prepared.path)
config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=prepared_raw, campaign_id=campaign.id)
@@ -166,86 +348,15 @@ def run_mock_campaign_send(
build_result = build_campaign_messages(config, campaign_file=prepared.path, write_eml=False)
files.annotate_built_messages_with_managed_files(build_result.built_messages, prepared.managed_files_by_local_path)
send_results: list[dict[str, Any]] = []
sent_count = 0
failed_count = 0
skipped_count = 0
imap_appended_count = 0
imap_failed_count = 0
for built in build_result.built_messages:
draft = built.draft
can_send, skip_reason = _can_mock_send(draft, include_warnings=include_warnings, include_needs_review=include_needs_review)
row: dict[str, Any] = {
"entry_index": draft.entry_index,
"entry_id": draft.entry_id,
"subject": draft.subject,
"validation_status": draft.validation_status.value,
"build_status": str(draft.build_status.value if hasattr(draft.build_status, "value") else draft.build_status),
"to": _message_addresses_payload(draft.to),
"attachments": _attachment_payloads(draft),
"issues": _issue_payloads(draft),
}
if not can_send or built.mime is None:
skipped_count += 1
row.update({"status": "skipped", "message": skip_reason or "Message has no MIME output"})
send_results.append(row)
continue
recipients = _recipient_emails(draft)
envelope_from = _envelope_from(draft)
if not recipients:
skipped_count += 1
row.update({"status": "skipped", "message": "No envelope recipients"})
send_results.append(row)
continue
if not send:
row.update({"status": "ready", "message": f"Would send to {len(recipients)} recipient(s)", "envelope_from": envelope_from, "envelope_recipients": recipients})
send_results.append(row)
continue
try:
if mailbox is not None and mailbox.consume_fail_next_smtp():
raise MockCampaignSendError("Configured mock failure: next SMTP delivery fails")
rejected = _smtp_rejection_matches(recipients)
if rejected and len(rejected) == len(recipients):
raise MockCampaignSendError(f"Configured mock failure: all recipients rejected ({', '.join(rejected)})")
accepted = [recipient for recipient in recipients if recipient not in rejected]
smtp_record = mailbox.record_smtp_delivery(built.mime, envelope_from=envelope_from, envelope_recipients=accepted, smtp_host="mock.smtp.local")
sent_count += 1
row.update({
"status": "sent",
"message": f"Mock SMTP captured as {smtp_record.id}",
"smtp_message_id": smtp_record.id,
"envelope_from": envelope_from,
"envelope_recipients": accepted,
"refused_recipients": rejected,
})
if append_sent:
try:
if mailbox is not None and mailbox.consume_fail_next_imap():
raise MockCampaignSendError("Configured mock failure: next IMAP append fails")
folder = "Sent"
if config.delivery.imap_append_sent.folder and config.delivery.imap_append_sent.folder != "auto":
folder = config.delivery.imap_append_sent.folder
elif config.server.imap and config.server.imap.sent_folder and config.server.imap.sent_folder != "auto":
folder = config.server.imap.sent_folder
imap_record = mailbox.record_imap_append(_raw_message_bytes(built.mime), folder=folder, imap_host="mock.imap.local")
imap_appended_count += 1
row.update({"imap_status": "appended", "imap_message_id": imap_record.id, "imap_folder": folder})
except Exception as exc:
imap_failed_count += 1
row.update({"imap_status": "failed", "imap_error": str(exc)})
except Exception as exc:
failed_count += 1
row.update({"status": "failed", "message": str(exc), "envelope_from": envelope_from, "envelope_recipients": recipients})
send_results.append(row)
send_batch = _mock_send_batch(
config=config,
built_messages=build_result.built_messages,
mailbox=mailbox,
send=send,
include_warnings=include_warnings,
include_needs_review=include_needs_review,
append_sent=append_sent,
)
validation_json = validation_report.model_dump(mode="json")
validation_json.update({"ok": validation_report.ok, "error_count": validation_report.error_count, "warning_count": validation_report.warning_count})
@@ -261,7 +372,6 @@ def run_mock_campaign_send(
"messages": [_message_payload(message) for message in build_report.messages],
})
attempted_count = sum(1 for row in send_results if row.get("status") in {"sent", "failed"})
return {
"campaign_id": campaign.id,
"version_id": version.id,
@@ -273,19 +383,19 @@ def run_mock_campaign_send(
"steps": [
{"key": "validate", "label": "Validate campaign JSON", "status": "ok" if validation_report.ok else "needs_review", "summary": validation_json},
{"key": "build", "label": "Build messages", "status": "ok" if build_report.queueable_count else "needs_review", "summary": {"built": build_report.built_count, "queueable": build_report.queueable_count, "needs_review": build_report.needs_review_count, "blocked": build_report.blocked_count}},
{"key": "send", "label": "Mock SMTP delivery", "status": "skipped" if not send else ("ok" if failed_count == 0 else "needs_review"), "summary": {"attempted": attempted_count, "sent": sent_count, "failed": failed_count, "skipped": skipped_count}},
{"key": "imap", "label": "Mock IMAP Sent append", "status": "skipped" if not send or not append_sent else ("ok" if imap_failed_count == 0 else "needs_review"), "summary": {"appended": imap_appended_count, "failed": imap_failed_count}},
{"key": "send", "label": "Mock SMTP delivery", "status": "skipped" if not send else ("ok" if send_batch.failed_count == 0 else "needs_review"), "summary": {"attempted": send_batch.attempted_count, "sent": send_batch.sent_count, "failed": send_batch.failed_count, "skipped": send_batch.skipped_count}},
{"key": "imap", "label": "Mock IMAP Sent append", "status": "skipped" if not send or not append_sent else ("ok" if send_batch.imap_failed_count == 0 else "needs_review"), "summary": {"appended": send_batch.imap_appended_count, "failed": send_batch.imap_failed_count}},
],
"validation": validation_json,
"build": build_json,
"send": {
"attempted_count": attempted_count,
"sent_count": sent_count,
"failed_count": failed_count,
"skipped_count": skipped_count,
"imap_appended_count": imap_appended_count,
"imap_failed_count": imap_failed_count,
"results": send_results,
"attempted_count": send_batch.attempted_count,
"sent_count": send_batch.sent_count,
"failed_count": send_batch.failed_count,
"skipped_count": send_batch.skipped_count,
"imap_appended_count": send_batch.imap_appended_count,
"imap_failed_count": send_batch.imap_failed_count,
"results": send_batch.results,
},
"mailbox": {"messages": mailbox.list_records(limit=200) if mailbox is not None else []},
}

View File

@@ -144,7 +144,7 @@ manifest = ModuleManifest(
name="Campaigns",
version="0.1.8",
required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
optional_dependencies=("files", "mail"),
optional_dependencies=("files", "mail", "notifications", "addresses"),
provides_interfaces=(
ModuleInterfaceProvider(name="campaigns.access", version="0.1.6"),
ModuleInterfaceProvider(name="campaigns.delivery_tasks", version="0.1.6"),
@@ -165,6 +165,18 @@ manifest = ModuleManifest(
version_max_exclusive="0.2.0",
optional=True,
),
ModuleInterfaceRequirement(
name="addresses.lookup",
version_min="0.1.0",
version_max_exclusive="0.2.0",
optional=True,
),
ModuleInterfaceRequirement(
name="addresses.recipient_source",
version_min="0.1.0",
version_max_exclusive="0.2.0",
optional=True,
),
),
permissions=PERMISSIONS,
route_factory=_campaigns_router,
@@ -175,7 +187,7 @@ manifest = ModuleManifest(
NavItem(
path="/operator",
label="Operator Queue",
icon="activity",
icon="radio-tower",
required_any=(
"campaigns:campaign:queue",
"campaigns:campaign:retry",
@@ -185,7 +197,7 @@ manifest = ModuleManifest(
),
order=30,
),
NavItem(path="/reports", label="Reports", icon="reports", required_any=("campaigns:report:read",), order=70),
NavItem(path="/reports", label="Reports", icon="clipboard-pen-line", required_any=("campaigns:report:read",), order=70),
),
frontend=FrontendModule(
module_id="campaigns",
@@ -195,7 +207,7 @@ manifest = ModuleManifest(
NavItem(
path="/operator",
label="Operator Queue",
icon="activity",
icon="radio-tower",
required_any=(
"campaigns:campaign:queue",
"campaigns:campaign:retry",
@@ -205,9 +217,8 @@ manifest = ModuleManifest(
),
order=30,
),
NavItem(path="/reports", label="Reports", icon="reports", required_any=("campaigns:report:read",), order=70),
NavItem(path="/address-book", label="Address Book", icon="users", order=80),
NavItem(path="/templates", label="Templates", icon="form", order=90),
NavItem(path="/reports", label="Reports", icon="clipboard-pen-line", required_any=("campaigns:report:read",), order=70),
NavItem(path="/templates", label="Templates", icon="layout-template", order=90),
),
),
migration_spec=MigrationSpec(

View File

@@ -81,6 +81,33 @@ class CampaignBuildResult:
built_messages: list[BuiltMessage]
@dataclass(slots=True)
class _EntryMessageContext:
resolution: EntryAttachmentResolution
senders: list[RecipientConfig]
sender: RecipientConfig | None
recipients: dict[str, list[RecipientConfig]]
issues: list[MessageIssue]
validation_status: MessageValidationStatus
@dataclass(slots=True)
class _RenderedMessageTemplate:
subject: str
text_body: str | None
html_body: str | None
body_mode: str
values: dict[str, Any]
@dataclass(slots=True)
class _MimeBuildResult:
message: EmailMessage | None
build_status: BuildStatus
validation_status: MessageValidationStatus
attachment_count: int
def _resolve(campaign_file: str | Path, raw_path: str) -> Path:
campaign_path = Path(campaign_file).resolve()
path = Path(raw_path).expanduser()
@@ -431,6 +458,322 @@ def _write_eml(message: EmailMessage, output_dir: Path, entry: EntryConfig, entr
return str(path), path.stat().st_size
def _entry_message_context(
*,
config: CampaignConfig,
campaign_file: str | Path,
entry: EntryConfig,
entry_index: int,
attachment_match_index: AttachmentMatchIndex | None,
) -> _EntryMessageContext:
resolution = resolve_entry_attachments(
config=config,
campaign_file=campaign_file,
entry=entry,
entry_index=entry_index,
match_index=attachment_match_index,
)
effective_addresses = effective_address_lists(config, entry)
senders = effective_addresses["from"]
sender = senders[0] if senders else None
recipients = {key: value for key, value in effective_addresses.items() if key != "from"}
issues = _message_issues_from_attachment_resolution(resolution)
validation_status = _validation_status_from_attachment_status(resolution.status)
validation_status = _apply_ignored_field_override_issues(config, entry, issues, validation_status)
return _EntryMessageContext(
resolution=resolution,
senders=senders,
sender=sender,
recipients=recipients,
issues=issues,
validation_status=validation_status,
)
def _apply_ignored_field_override_issues(
config: CampaignConfig,
entry: EntryConfig,
issues: list[MessageIssue],
validation_status: MessageValidationStatus,
) -> MessageValidationStatus:
ignored_field_overrides = ignored_entry_field_overrides(config, entry)
if not ignored_field_overrides:
return validation_status
issues.append(
MessageIssue(
severity="warning",
code="field_override_not_allowed",
message=(
"Recipient field value(s) ignored because the campaign field does not allow overrides: "
+ ", ".join(ignored_field_overrides)
),
behavior="warn",
source="fields",
)
)
if validation_status == MessageValidationStatus.READY:
return MessageValidationStatus.WARNING
return validation_status
def _message_draft(
*,
config: CampaignConfig,
entry: EntryConfig,
entry_index: int,
context: _EntryMessageContext,
build_status: BuildStatus,
validation_status: MessageValidationStatus,
send_status: SendStatus = SendStatus.DRAFT,
imap_status: ImapStatus | None = None,
subject: str | None = None,
attachment_count: int = 0,
issues: list[MessageIssue] | None = None,
eml_path: str | None = None,
eml_size: int | None = None,
) -> MessageDraft:
if imap_status is None:
imap_status = _imap_initial_status(config) if build_status == BuildStatus.BUILT else ImapStatus.SKIPPED
return MessageDraft(
entry_index=entry_index,
entry_id=entry.id,
active=entry.active,
build_status=build_status,
validation_status=validation_status,
send_status=send_status,
imap_status=imap_status,
subject=subject,
from_=_message_address(context.sender),
from_all=_message_addresses(context.senders),
to=_message_addresses(context.recipients["to"]),
cc=_message_addresses(context.recipients["cc"]),
bcc=_message_addresses(context.recipients["bcc"]),
reply_to=_message_addresses(context.recipients["reply_to"]),
bounce_to=_message_addresses(context.recipients["bounce_to"]),
disposition_notification_to=_message_addresses(context.recipients["disposition_notification_to"]),
attachment_count=attachment_count,
attachments=_attachment_summaries(context.resolution),
issues=issues if issues is not None else context.issues,
eml_path=eml_path,
eml_size_bytes=eml_size,
)
def _inactive_entry_message(
*,
config: CampaignConfig,
entry: EntryConfig,
entry_index: int,
context: _EntryMessageContext,
) -> BuiltMessage:
draft = _message_draft(
config=config,
entry=entry,
entry_index=entry_index,
context=context,
build_status=BuildStatus.BUILD_FAILED,
validation_status=MessageValidationStatus.INACTIVE,
imap_status=ImapStatus.SKIPPED,
issues=[
MessageIssue(
severity="info",
code="inactive_entry",
message="Entry is inactive",
behavior=config.validation_policy.inactive_entry.value,
source="entry",
)
],
)
return BuiltMessage(draft=draft, mime=None)
def _validate_required_recipients(
config: CampaignConfig,
recipients: dict[str, list[RecipientConfig]],
issues: list[MessageIssue],
validation_status: MessageValidationStatus,
) -> MessageValidationStatus:
if recipients["to"]:
return validation_status
behavior = config.validation_policy.missing_email.value
issues.append(
_issue_from_behavior(
code="missing_email",
message="No effective To recipient is configured",
behavior=behavior,
source="recipients",
)
)
if behavior == MissingAddressBehavior.BLOCK.value:
return MessageValidationStatus.BLOCKED
return MessageValidationStatus.EXCLUDED
def _render_message_template(
config: CampaignConfig,
campaign_file: str | Path,
entry: EntryConfig,
) -> _RenderedMessageTemplate:
subject_template, text_template, html_template = _load_template_parts(config, campaign_file)
body_mode = _template_body_mode(config)
values = build_template_values(config, entry)
keep_missing_placeholders = not config.validation_policy.ignore_empty_fields
subject = _render_template(subject_template, values, keep_missing=keep_missing_placeholders)
text_body = None
html_body = None
if text_template is not None:
text_body = _render_template(text_template or "", values, keep_missing=keep_missing_placeholders)
if html_template is not None:
html_body = _render_template(html_template or "", values, keep_missing=keep_missing_placeholders)
return _RenderedMessageTemplate(
subject=subject,
text_body=text_body,
html_body=html_body,
body_mode=body_mode,
values=values,
)
def _unresolved_template_fields(rendered: _RenderedMessageTemplate) -> list[str]:
unresolved_fields = _find_unresolved_placeholders(rendered.subject)
if rendered.body_mode in {TemplateBodyMode.TEXT.value, TemplateBodyMode.BOTH.value}:
unresolved_fields |= _find_unresolved_placeholders(rendered.text_body)
if rendered.body_mode in {TemplateBodyMode.HTML.value, TemplateBodyMode.BOTH.value}:
unresolved_fields |= _find_unresolved_placeholders(rendered.html_body)
return sorted(unresolved_fields)
def _validate_rendered_template(
config: CampaignConfig,
rendered: _RenderedMessageTemplate,
issues: list[MessageIssue],
validation_status: MessageValidationStatus,
) -> MessageValidationStatus:
unresolved = _unresolved_template_fields(rendered)
if not unresolved:
return validation_status
behavior = config.validation_policy.template_error.value
issues.append(
_issue_from_behavior(
code="template_error",
message="Unresolved template placeholder(s): " + ", ".join(unresolved),
behavior=behavior,
source="template",
)
)
if behavior == MissingAddressBehavior.BLOCK.value:
return MessageValidationStatus.BLOCKED
return MessageValidationStatus.EXCLUDED
def _populate_message_headers(
message: EmailMessage,
*,
senders: list[RecipientConfig],
recipients: dict[str, list[RecipientConfig]],
subject: str,
) -> None:
message["Date"] = formatdate(localtime=True)
message["Message-ID"] = make_msgid()
if senders:
message["From"] = _format_recipient_header(senders)
if len(senders) > 1:
# RFC 5322 requires a singular Sender when From contains more
# than one mailbox. The first effective From address remains
# the SMTP envelope sender and compatibility primary value.
message["Sender"] = _format_recipient(senders[0])
if recipients["to"]:
message["To"] = _format_recipient_header(recipients["to"])
if recipients["cc"]:
message["Cc"] = _format_recipient_header(recipients["cc"])
# Bcc deliberately remains envelope-only and is tracked in MessageDraft.
if recipients["reply_to"]:
message["Reply-To"] = _format_recipient_header(recipients["reply_to"])
if recipients["disposition_notification_to"]:
message["Disposition-Notification-To"] = _format_recipient_header(
recipients["disposition_notification_to"]
)
# bounce_to is tracked but not emitted as Return-Path. That should be the SMTP envelope sender.
message["Subject"] = subject
def _populate_message_body(message: EmailMessage, rendered: _RenderedMessageTemplate) -> None:
effective_html_body = rendered.html_body if rendered.html_body and rendered.html_body.strip() else None
if rendered.body_mode == TemplateBodyMode.HTML.value:
message.set_content(effective_html_body or "", subtype="html")
elif rendered.body_mode == TemplateBodyMode.TEXT.value:
message.set_content(rendered.text_body or "")
elif effective_html_body is not None:
message.set_content(rendered.text_body or "")
message.add_alternative(effective_html_body, subtype="html")
else:
message.set_content(rendered.text_body or "")
def _build_mime_message(
*,
config: CampaignConfig,
entry: EntryConfig,
entry_index: int,
output_dir: Path | None,
work_dir: Path | None,
context: _EntryMessageContext,
rendered: _RenderedMessageTemplate,
) -> _MimeBuildResult:
message = EmailMessage()
try:
_populate_message_headers(
message,
senders=context.senders,
recipients=context.recipients,
subject=rendered.subject,
)
_populate_message_body(message, rendered)
if work_dir is None:
work_dir = output_dir or Path(tempfile.mkdtemp(prefix="govoplan-build-"))
attachment_count = _attach_files(
message=message,
config=config,
entry=entry,
entry_index=entry_index,
resolution=context.resolution,
values=rendered.values,
work_dir=work_dir,
)
return _MimeBuildResult(
message=message,
build_status=BuildStatus.BUILT,
validation_status=context.validation_status,
attachment_count=attachment_count,
)
except ZipBuildError as exc:
context.issues.append(
MessageIssue(
severity="error",
code=exc.code,
message=str(exc),
behavior="block",
source="attachments",
)
)
except Exception as exc:
context.issues.append(
MessageIssue(
severity="error",
code="build_failed",
message=str(exc),
behavior="block",
source="builder",
)
)
return _MimeBuildResult(
message=None,
build_status=BuildStatus.BUILD_FAILED,
validation_status=MessageValidationStatus.BLOCKED,
attachment_count=0,
)
def build_entry_message(
*,
config: CampaignConfig,
@@ -442,169 +785,57 @@ def build_entry_message(
work_dir: Path | None = None,
attachment_match_index: AttachmentMatchIndex | None = None,
) -> BuiltMessage:
resolution = resolve_entry_attachments(config=config, campaign_file=campaign_file, entry=entry, entry_index=entry_index, match_index=attachment_match_index)
effective_addresses = effective_address_lists(config, entry)
senders = effective_addresses["from"]
sender = senders[0] if senders else None
recipients = {key: value for key, value in effective_addresses.items() if key != "from"}
issues = _message_issues_from_attachment_resolution(resolution)
validation_status = _validation_status_from_attachment_status(resolution.status)
ignored_field_overrides = ignored_entry_field_overrides(config, entry)
if ignored_field_overrides:
issues.append(
MessageIssue(
severity="warning",
code="field_override_not_allowed",
message="Recipient field value(s) ignored because the campaign field does not allow overrides: " + ", ".join(ignored_field_overrides),
behavior="warn",
source="fields",
)
)
if validation_status == MessageValidationStatus.READY:
validation_status = MessageValidationStatus.WARNING
context = _entry_message_context(
config=config,
campaign_file=campaign_file,
entry=entry,
entry_index=entry_index,
attachment_match_index=attachment_match_index,
)
if not entry.active:
draft = MessageDraft(
entry_index=entry_index,
entry_id=entry.id,
active=False,
build_status=BuildStatus.BUILD_FAILED,
validation_status=MessageValidationStatus.INACTIVE,
send_status=SendStatus.DRAFT,
imap_status=ImapStatus.SKIPPED,
from_=_message_address(sender),
from_all=_message_addresses(senders),
to=_message_addresses(recipients["to"]),
cc=_message_addresses(recipients["cc"]),
bcc=_message_addresses(recipients["bcc"]),
reply_to=_message_addresses(recipients["reply_to"]),
bounce_to=_message_addresses(recipients["bounce_to"]),
disposition_notification_to=_message_addresses(recipients["disposition_notification_to"]),
attachments=_attachment_summaries(resolution),
issues=[MessageIssue(severity="info", code="inactive_entry", message="Entry is inactive", behavior=config.validation_policy.inactive_entry.value, source="entry")],
)
return BuiltMessage(draft=draft, mime=None)
return _inactive_entry_message(config=config, entry=entry, entry_index=entry_index, context=context)
if not recipients["to"]:
behavior = config.validation_policy.missing_email.value
issues.append(_issue_from_behavior(code="missing_email", message="No effective To recipient is configured", behavior=behavior, source="recipients"))
validation_status = MessageValidationStatus.BLOCKED if behavior == MissingAddressBehavior.BLOCK.value else MessageValidationStatus.EXCLUDED
subject_template, text_template, html_template = _load_template_parts(config, campaign_file)
body_mode = _template_body_mode(config)
values = build_template_values(config, entry)
keep_missing_placeholders = not config.validation_policy.ignore_empty_fields
subject = _render_template(subject_template, values, keep_missing=keep_missing_placeholders)
text_body = _render_template(text_template or "", values, keep_missing=keep_missing_placeholders) if text_template is not None else None
html_body = _render_template(html_template or "", values, keep_missing=keep_missing_placeholders) if html_template is not None else None
unresolved_fields = _find_unresolved_placeholders(subject)
if body_mode in {TemplateBodyMode.TEXT.value, TemplateBodyMode.BOTH.value}:
unresolved_fields |= _find_unresolved_placeholders(text_body)
if body_mode in {TemplateBodyMode.HTML.value, TemplateBodyMode.BOTH.value}:
unresolved_fields |= _find_unresolved_placeholders(html_body)
unresolved = sorted(unresolved_fields)
if unresolved:
behavior = config.validation_policy.template_error.value
issues.append(
_issue_from_behavior(
code="template_error",
message="Unresolved template placeholder(s): " + ", ".join(unresolved),
behavior=behavior,
source="template",
)
)
validation_status = MessageValidationStatus.BLOCKED if behavior == MissingAddressBehavior.BLOCK.value else MessageValidationStatus.EXCLUDED
message = EmailMessage()
try:
message["Date"] = formatdate(localtime=True)
message["Message-ID"] = make_msgid()
if senders:
message["From"] = _format_recipient_header(senders)
if len(senders) > 1:
# RFC 5322 requires a singular Sender when From contains more
# than one mailbox. The first effective From address remains
# the SMTP envelope sender and compatibility primary value.
message["Sender"] = _format_recipient(senders[0])
if recipients["to"]:
message["To"] = _format_recipient_header(recipients["to"])
if recipients["cc"]:
message["Cc"] = _format_recipient_header(recipients["cc"])
# Bcc deliberately remains envelope-only and is tracked in MessageDraft.
if recipients["reply_to"]:
message["Reply-To"] = _format_recipient_header(recipients["reply_to"])
if recipients["disposition_notification_to"]:
message["Disposition-Notification-To"] = _format_recipient_header(recipients["disposition_notification_to"])
# bounce_to is tracked but not emitted as Return-Path. That should be the SMTP envelope sender.
message["Subject"] = subject
effective_html_body = html_body if html_body and html_body.strip() else None
if body_mode == TemplateBodyMode.HTML.value:
message.set_content(effective_html_body or "", subtype="html")
elif body_mode == TemplateBodyMode.TEXT.value:
message.set_content(text_body or "")
elif effective_html_body is not None:
message.set_content(text_body or "")
message.add_alternative(effective_html_body, subtype="html")
else:
message.set_content(text_body or "")
if work_dir is None:
work_dir = output_dir or Path(tempfile.mkdtemp(prefix="multimailer-build-"))
attachment_count = _attach_files(
message=message,
config=config,
entry=entry,
entry_index=entry_index,
resolution=resolution,
values=values,
work_dir=work_dir,
)
build_status = BuildStatus.BUILT
except ZipBuildError as exc:
issues.append(MessageIssue(severity="error", code=exc.code, message=str(exc), behavior="block", source="attachments"))
validation_status = MessageValidationStatus.BLOCKED
build_status = BuildStatus.BUILD_FAILED
attachment_count = 0
message = None # type: ignore[assignment]
except Exception as exc:
issues.append(MessageIssue(severity="error", code="build_failed", message=str(exc), behavior="block", source="builder"))
validation_status = MessageValidationStatus.BLOCKED
build_status = BuildStatus.BUILD_FAILED
attachment_count = 0
message = None # type: ignore[assignment]
context.validation_status = _validate_required_recipients(
config,
context.recipients,
context.issues,
context.validation_status,
)
rendered = _render_message_template(config, campaign_file, entry)
context.validation_status = _validate_rendered_template(
config,
rendered,
context.issues,
context.validation_status,
)
mime_result = _build_mime_message(
config=config,
entry=entry,
entry_index=entry_index,
output_dir=output_dir,
work_dir=work_dir,
context=context,
rendered=rendered,
)
eml_path: str | None = None
eml_size: int | None = None
if write_eml and output_dir is not None and message is not None:
eml_path, eml_size = _write_eml(message, output_dir, entry, entry_index)
if write_eml and output_dir is not None and mime_result.message is not None:
eml_path, eml_size = _write_eml(mime_result.message, output_dir, entry, entry_index)
draft = MessageDraft(
draft = _message_draft(
config=config,
entry=entry,
entry_index=entry_index,
entry_id=entry.id,
active=entry.active,
build_status=build_status,
validation_status=validation_status,
send_status=SendStatus.DRAFT,
imap_status=_imap_initial_status(config) if build_status == BuildStatus.BUILT else ImapStatus.SKIPPED,
subject=subject,
from_=_message_address(sender),
from_all=_message_addresses(senders),
to=_message_addresses(recipients["to"]),
cc=_message_addresses(recipients["cc"]),
bcc=_message_addresses(recipients["bcc"]),
reply_to=_message_addresses(recipients["reply_to"]),
bounce_to=_message_addresses(recipients["bounce_to"]),
disposition_notification_to=_message_addresses(recipients["disposition_notification_to"]),
attachment_count=attachment_count,
attachments=_attachment_summaries(resolution),
issues=issues,
context=context,
build_status=mime_result.build_status,
validation_status=mime_result.validation_status,
subject=rendered.subject,
attachment_count=mime_result.attachment_count,
eml_path=eml_path,
eml_size_bytes=eml_size,
eml_size=eml_size,
)
return BuiltMessage(draft=draft, mime=message)
return BuiltMessage(draft=draft, mime=mime_result.message)
@@ -680,7 +911,7 @@ def build_campaign_messages(
started = time.perf_counter()
attachment_match_index = AttachmentMatchIndex()
with tempfile.TemporaryDirectory(prefix="multimailer-build-") as tmp:
with tempfile.TemporaryDirectory(prefix="govoplan-build-") as tmp:
work_dir = output_path or Path(tmp)
built_messages = [
build_entry_message(

View File

@@ -275,7 +275,7 @@ def validate_campaign_version(
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=False,
prefix="multimailer-managed-validate-",
prefix="govoplan-managed-validate-",
) as prepared:
managed_raw = load_campaign_json(prepared.path)
managed_config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=managed_raw, campaign_id=campaign.id)
@@ -407,7 +407,7 @@ def build_campaign_version(
campaign_id=campaign.id,
raw_json=version.raw_json if isinstance(version.raw_json, dict) else {},
include_bytes=True,
prefix="multimailer-managed-build-",
prefix="govoplan-managed-build-",
) as prepared:
managed_raw = load_campaign_json(prepared.path)
managed_config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=managed_raw, campaign_id=campaign.id)

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import copy
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
@@ -31,6 +32,39 @@ class LockedCampaignVersionError(CampaignPersistenceError):
"""Raised when a caller tries to edit an immutable campaign version."""
@dataclass(slots=True)
class _PartialValidationCollector:
section: str | None
issues: list[dict[str, Any]] = field(default_factory=list)
def issue(self, severity: str, section: str, field: str, code: str, message: str) -> None:
if self.section is None or self.section == section:
self.issues.append({
"severity": severity,
"section": section,
"field": field,
"code": code,
"message": message,
})
def result(self) -> dict[str, Any]:
return {
"ok": not any(item["severity"] == "error" for item in self.issues),
"section": self.section,
"error_count": sum(1 for item in self.issues if item["severity"] == "error"),
"warning_count": sum(1 for item in self.issues if item["severity"] == "warning"),
"info_count": sum(1 for item in self.issues if item["severity"] == "info"),
"issues": self.issues,
}
def _require_campaign(session: Session, campaign_id: str) -> Campaign:
campaign = session.get(Campaign, campaign_id)
if campaign is None:
raise CampaignPersistenceError(f"Campaign not found: {campaign_id}")
return campaign
USER_LOCK_TEMPORARY = "temporary"
USER_LOCK_PERMANENT = "permanent"
USER_LOCK_STATES = {USER_LOCK_TEMPORARY, USER_LOCK_PERMANENT}
@@ -320,8 +354,7 @@ def fork_campaign_version_for_edit(
"""
source = get_campaign_version_for_tenant(session, tenant_id=tenant_id, campaign_id=campaign_id, version_id=version_id)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
if campaign_has_active_working_version(session, campaign):
current = session.get(CampaignVersion, campaign.current_version_id)
@@ -429,8 +462,7 @@ def unlock_validated_campaign_version(
"""
version = get_campaign_version_for_tenant(session, tenant_id=tenant_id, campaign_id=campaign_id, version_id=version_id)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
ensure_current_working_version(campaign, version, action="unlock")
if is_temporary_user_locked_version(version):
@@ -490,8 +522,7 @@ def update_campaign_version(
autosave: bool = False,
) -> CampaignVersion:
version = get_campaign_version_for_tenant(session, tenant_id=tenant_id, campaign_id=campaign_id, version_id=version_id)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
ensure_current_working_version(campaign, version, action="edit")
if is_version_locked(version):
@@ -566,64 +597,95 @@ def update_campaign_review_state(
campaign_id=campaign_id,
version_id=version_id,
)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
ensure_current_working_version(campaign, version, action="record review state for")
if is_version_final_locked(version):
raise LockedCampaignVersionError("Delivery has started; message review state can no longer be changed.")
build_token = _campaign_review_build_token(version)
normalized_reviewed = list(dict.fromkeys(str(value) for value in reviewed_message_keys if str(value).strip()))
if inspection_complete:
normalized_reviewed = _complete_campaign_review_keys(session, version, normalized_reviewed)
_write_campaign_review_state(
version,
build_token=build_token,
inspection_complete=inspection_complete,
reviewed_message_keys=normalized_reviewed,
user_id=user_id,
)
session.add(version)
session.commit()
return version
def _campaign_review_build_token(version: CampaignVersion) -> str:
build_summary = version.build_summary if isinstance(version.build_summary, dict) else {}
if not build_summary:
raise CampaignPersistenceError("Build messages before recording review state.")
build_token = str(build_summary.get("build_token") or build_summary.get("built_at") or "").strip()
if not build_token:
# Backwards-compatible upgrade for build summaries created before
# review-state tokens were introduced.
build_token = uuid4().hex
build_summary = copy.deepcopy(build_summary)
build_summary["build_token"] = build_token
version.build_summary = build_summary
if build_token:
return build_token
build_token = uuid4().hex
updated_summary = copy.deepcopy(build_summary)
updated_summary["build_token"] = build_token
version.build_summary = updated_summary
return build_token
normalized_reviewed = list(dict.fromkeys(str(value) for value in reviewed_message_keys if str(value).strip()))
if inspection_complete:
jobs = (
session.query(CampaignJob)
.filter(CampaignJob.campaign_version_id == version.id)
.order_by(CampaignJob.entry_index.asc())
.all()
def _complete_campaign_review_keys(
session: Session,
version: CampaignVersion,
reviewed_message_keys: list[str],
) -> list[str]:
jobs = (
session.query(CampaignJob)
.filter(CampaignJob.campaign_version_id == version.id)
.order_by(CampaignJob.entry_index.asc())
.all()
)
blocking = [job for job in jobs if job.build_status != "built" or job.validation_status == "blocked"]
if blocking:
raise CampaignPersistenceError("Blocked or failed messages must be resolved before review can be completed.")
missing = sorted(_required_review_keys(jobs) - set(reviewed_message_keys))
if missing:
raise CampaignPersistenceError(
"Messages requiring an explicit decision must be opened before review can be completed: " + ", ".join(missing)
)
blocking = [job for job in jobs if job.build_status != "built" or job.validation_status == "blocked"]
if blocking:
raise CampaignPersistenceError("Blocked or failed messages must be resolved before review can be completed.")
reviewed = set(normalized_reviewed)
explicit = {
str(job.entry_id or job.entry_index)
for job in jobs
if job.validation_status == "needs_review"
}
missing = sorted(explicit - reviewed)
if missing:
raise CampaignPersistenceError(
"Messages requiring an explicit decision must be opened before review can be completed: " + ", ".join(missing)
)
bulk_acceptable = [
str(job.entry_id or job.entry_index)
for job in jobs
if job.validation_status in {"warning", "excluded"}
]
normalized_reviewed = list(dict.fromkeys([*normalized_reviewed, *bulk_acceptable]))
return list(dict.fromkeys([*reviewed_message_keys, *_bulk_acceptable_review_keys(jobs)]))
def _required_review_keys(jobs: list[CampaignJob]) -> set[str]:
return {
str(job.entry_id or job.entry_index)
for job in jobs
if job.validation_status == "needs_review"
}
def _bulk_acceptable_review_keys(jobs: list[CampaignJob]) -> list[str]:
return [
str(job.entry_id or job.entry_index)
for job in jobs
if job.validation_status in {"warning", "excluded"}
]
def _write_campaign_review_state(
version: CampaignVersion,
*,
build_token: str,
inspection_complete: bool,
reviewed_message_keys: list[str],
user_id: str | None,
) -> None:
editor_state = copy.deepcopy(version.editor_state or {})
editor_state["review_send"] = {
"build_token": build_token,
"inspection_complete": bool(inspection_complete),
"reviewed_message_keys": normalized_reviewed,
"reviewed_message_keys": reviewed_message_keys,
"updated_at": datetime.now(UTC).isoformat(),
"updated_by_user_id": user_id,
}
version.editor_state = editor_state
session.add(version)
session.commit()
return version
def lock_campaign_version_temporarily(
@@ -642,8 +704,7 @@ def lock_campaign_version_temporarily(
campaign_id=campaign_id,
version_id=version_id,
)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
ensure_current_working_version(campaign, version, action="lock")
if is_version_final_locked(version):
raise LockedCampaignVersionError("Delivery/final versions are permanently locked and cannot receive a temporary user lock.")
@@ -677,8 +738,7 @@ def unlock_user_locked_campaign_version(
campaign_id=campaign_id,
version_id=version_id,
)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
ensure_current_working_version(campaign, version, action="unlock")
state = campaign_version_user_lock_state(version)
if state == USER_LOCK_PERMANENT:
@@ -716,8 +776,7 @@ def permanently_lock_campaign_version(
campaign_id=campaign_id,
version_id=version_id,
)
campaign = session.get(Campaign, campaign_id)
assert campaign is not None
campaign = _require_campaign(session, campaign_id)
ensure_current_working_version(campaign, version, action="lock permanently")
if is_version_final_locked(version):
raise LockedCampaignVersionError("This version is already permanently locked by its delivery/final state.")
@@ -761,79 +820,101 @@ def validate_campaign_partial(raw_json: dict[str, Any], *, section: str | None =
lets the WebUI autosave and validate one wizard step at a time.
"""
issues: list[dict[str, Any]] = []
collector = _PartialValidationCollector(section=section)
_validate_partial_basics(collector, _dict_value(raw_json, "campaign"))
recipients = _dict_value(raw_json, "recipients")
_validate_partial_sender(collector, recipients)
_validate_partial_recipients(collector, _dict_value(raw_json, "entries"))
_validate_partial_template(collector, _dict_value(raw_json, "template"))
_validate_partial_attachments(collector, _dict_value(raw_json, "attachments"))
_validate_partial_delivery(collector, _dict_value(raw_json, "delivery"))
return collector.result()
def issue(severity: str, sec: str, field: str, code: str, message: str) -> None:
if section is None or section == sec:
issues.append({
"severity": severity,
"section": sec,
"field": field,
"code": code,
"message": message,
})
campaign = raw_json.get("campaign") if isinstance(raw_json.get("campaign"), dict) else {}
def _dict_value(value: dict[str, Any], key: str) -> dict[str, Any]:
candidate = value.get(key)
return candidate if isinstance(candidate, dict) else {}
def _validate_partial_basics(collector: _PartialValidationCollector, campaign: dict[str, Any]) -> None:
if not campaign.get("id"):
issue("error", "basics", "campaign.id", "missing_campaign_id", "Campaign id is required.")
collector.issue("error", "basics", "campaign.id", "missing_campaign_id", "Campaign id is required.")
if not campaign.get("name"):
issue("error", "basics", "campaign.name", "missing_campaign_name", "Campaign name is required.")
collector.issue("error", "basics", "campaign.name", "missing_campaign_name", "Campaign name is required.")
recipients = raw_json.get("recipients") if isinstance(raw_json.get("recipients"), dict) else {}
def _validate_partial_sender(collector: _PartialValidationCollector, recipients: dict[str, Any]) -> None:
sender = recipients.get("from") if isinstance(recipients.get("from"), dict) else {}
if not sender.get("email"):
issue("warning", "sender", "recipients.from.email", "missing_sender_email", "Sender email is not configured yet.")
collector.issue("warning", "sender", "recipients.from.email", "missing_sender_email", "Sender email is not configured yet.")
entries = raw_json.get("entries") if isinstance(raw_json.get("entries"), dict) else {}
def _validate_partial_recipients(collector: _PartialValidationCollector, entries: dict[str, Any]) -> None:
has_inline = bool(entries.get("inline"))
has_source = isinstance(entries.get("source"), dict)
if not has_inline and not has_source:
issue("warning", "recipients", "entries", "missing_recipients", "No inline recipients or external recipient source configured yet.")
if has_source:
mapping = entries.get("mapping") if isinstance(entries.get("mapping"), dict) else {}
if not any(key in mapping for key in ("to.0.email", "to.email", "email")):
issue("warning", "recipients", "entries.mapping", "missing_email_mapping", "No email field mapping is configured.")
collector.issue("warning", "recipients", "entries", "missing_recipients", "No inline recipients or external recipient source configured yet.")
if has_source and not _entries_source_has_email_mapping(entries):
collector.issue("warning", "recipients", "entries.mapping", "missing_email_mapping", "No email field mapping is configured.")
template = raw_json.get("template") if isinstance(raw_json.get("template"), dict) else {}
def _entries_source_has_email_mapping(entries: dict[str, Any]) -> bool:
mapping = entries.get("mapping") if isinstance(entries.get("mapping"), dict) else {}
return any(key in mapping for key in ("to.0.email", "to.email", "email"))
def _validate_partial_template(collector: _PartialValidationCollector, template: dict[str, Any]) -> None:
source_template = template.get("source") if isinstance(template.get("source"), dict) else {}
if not template.get("subject") and not source_template.get("subject_path"):
issue("warning", "template", "template.subject", "missing_subject", "Template subject is empty.")
explicit_body_mode = template.get("body_mode") if template.get("body_mode") in {"text", "html", "both"} else None
has_text_body = bool(template.get("text")) or bool(source_template.get("text_path"))
has_html_body = bool(template.get("html")) or bool(source_template.get("html_path"))
if explicit_body_mode == "text" and not has_text_body:
issue("warning", "template", "template.text", "missing_template_text_body", "Template body mode is text only, but no text body is configured.")
elif explicit_body_mode == "html" and not has_html_body:
issue("warning", "template", "template.html", "missing_template_html_body", "Template body mode is HTML only, but no HTML body is configured.")
elif explicit_body_mode == "both":
if not has_text_body:
issue("warning", "template", "template.text", "missing_template_text_body", "Template body mode is both, but no text body is configured.")
if not has_html_body:
issue("warning", "template", "template.html", "missing_template_html_body", "Template body mode is both, but no HTML body is configured.")
elif not has_text_body and not has_html_body and not source_template:
issue("warning", "template", "template", "missing_template_body", "No text, HTML or file-based template body configured yet.")
collector.issue("warning", "template", "template.subject", "missing_subject", "Template subject is empty.")
body_state = _partial_template_body_state(template, source_template)
_validate_partial_template_body(collector, body_state)
attachments = raw_json.get("attachments") if isinstance(raw_json.get("attachments"), dict) else {}
def _partial_template_body_state(template: dict[str, Any], source_template: dict[str, Any]) -> dict[str, Any]:
return {
"mode": template.get("body_mode") if template.get("body_mode") in {"text", "html", "both"} else None,
"has_text": bool(template.get("text")) or bool(source_template.get("text_path")),
"has_html": bool(template.get("html")) or bool(source_template.get("html_path")),
"has_source": bool(source_template),
}
def _validate_partial_template_body(collector: _PartialValidationCollector, state: dict[str, Any]) -> None:
mode = state["mode"]
has_text = bool(state["has_text"])
has_html = bool(state["has_html"])
if mode == "text" and not has_text:
collector.issue("warning", "template", "template.text", "missing_template_text_body", "Template body mode is text only, but no text body is configured.")
elif mode == "html" and not has_html:
collector.issue("warning", "template", "template.html", "missing_template_html_body", "Template body mode is HTML only, but no HTML body is configured.")
elif mode == "both":
_validate_partial_dual_body_template(collector, has_text=has_text, has_html=has_html)
elif not has_text and not has_html and not state["has_source"]:
collector.issue("warning", "template", "template", "missing_template_body", "No text, HTML or file-based template body configured yet.")
def _validate_partial_dual_body_template(collector: _PartialValidationCollector, *, has_text: bool, has_html: bool) -> None:
if not has_text:
collector.issue("warning", "template", "template.text", "missing_template_text_body", "Template body mode is both, but no text body is configured.")
if not has_html:
collector.issue("warning", "template", "template.html", "missing_template_html_body", "Template body mode is both, but no HTML body is configured.")
def _validate_partial_attachments(collector: _PartialValidationCollector, attachments: dict[str, Any]) -> None:
base_paths = attachments.get("base_paths") if isinstance(attachments.get("base_paths"), list) else []
has_named_base_path = any(isinstance(item, dict) and item.get("path") for item in base_paths)
if not has_named_base_path and not attachments.get("base_path"):
issue("info", "attachments", "attachments.base_path", "missing_attachment_base_path", "Attachment base path is not configured yet.")
collector.issue("info", "attachments", "attachments.base_path", "missing_attachment_base_path", "Attachment base path is not configured yet.")
delivery = raw_json.get("delivery") if isinstance(raw_json.get("delivery"), dict) else {}
def _validate_partial_delivery(collector: _PartialValidationCollector, delivery: dict[str, Any]) -> None:
rate_limit = delivery.get("rate_limit") if isinstance(delivery.get("rate_limit"), dict) else {}
messages_per_minute = rate_limit.get("messages_per_minute")
if messages_per_minute is not None:
try:
if int(messages_per_minute) < 1:
issue("error", "send", "delivery.rate_limit.messages_per_minute", "invalid_rate_limit", "Messages per minute must be at least 1.")
except (TypeError, ValueError):
issue("error", "send", "delivery.rate_limit.messages_per_minute", "invalid_rate_limit", "Messages per minute must be a number.")
return {
"ok": not any(item["severity"] == "error" for item in issues),
"section": section,
"error_count": sum(1 for item in issues if item["severity"] == "error"),
"warning_count": sum(1 for item in issues if item["severity"] == "warning"),
"info_count": sum(1 for item in issues if item["severity"] == "info"),
"issues": issues,
}
if messages_per_minute is None:
return
try:
if int(messages_per_minute) < 1:
collector.issue("error", "send", "delivery.rate_limit.messages_per_minute", "invalid_rate_limit", "Messages per minute must be at least 1.")
except (TypeError, ValueError):
collector.issue("error", "send", "delivery.rate_limit.messages_per_minute", "invalid_rate_limit", "Messages per minute must be a number.")

View File

@@ -367,38 +367,127 @@ def generate_campaign_report(
campaign = _get_campaign(session, tenant_id=tenant_id, campaign_id=campaign_id)
version = _selected_version(session, campaign, version_id)
jobs = _report_jobs(session, tenant_id=tenant_id, campaign_id=campaign.id, version=version)
report = _campaign_report_payload(
session,
campaign=campaign,
version=version,
jobs=jobs,
tenant_id=tenant_id,
include_recent_failures=include_recent_failures,
)
if include_jobs:
report["jobs"] = [_job_row(job) for job in jobs]
return report
def _report_jobs(
session: Session,
*,
tenant_id: str,
campaign_id: str,
version: CampaignVersion | None,
) -> list[CampaignJob]:
jobs_query = session.query(CampaignJob).filter(
CampaignJob.tenant_id == tenant_id,
CampaignJob.campaign_id == campaign.id,
CampaignJob.campaign_id == campaign_id,
)
if version:
jobs_query = jobs_query.filter(CampaignJob.campaign_version_id == version.id)
else:
jobs_query = jobs_query.filter(False)
jobs = (
jobs_query
.order_by(CampaignJob.entry_index.asc())
.all()
)
return jobs_query.order_by(CampaignJob.entry_index.asc()).all()
def _campaign_report_payload(
session: Session,
*,
campaign: Campaign,
version: CampaignVersion | None,
jobs: list[CampaignJob],
tenant_id: str,
include_recent_failures: bool,
) -> dict[str, Any]:
job_ids = [job.id for job in jobs]
send_attempts = session.query(SendAttempt).filter(SendAttempt.job_id.in_(job_ids)).count() if job_ids else 0
imap_attempts = session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id.in_(job_ids)).count() if job_ids else 0
report = {
"generated_at": _utcnow_iso(),
"campaign": _campaign_report_campaign_payload(campaign),
"current_version": _version_info(version),
"selected_version_id": version.id if version else None,
"cards": _campaign_report_cards(version, jobs),
"status_counts": _campaign_report_status_counts(version, jobs),
"issues": {
**_issue_summary_from_jobs(jobs),
"persisted_campaign_issue_count": _persisted_campaign_issue_count(
session,
tenant_id=tenant_id,
campaign_id=campaign.id,
version=version,
),
},
"attachments": _attachment_summary(jobs),
"attempts": _campaign_report_attempt_counts(session, job_ids),
"delivery": _load_delivery_info(version, jobs),
}
if include_recent_failures:
report["recent_failures"] = _recent_failures(jobs)
return report
def _campaign_report_campaign_payload(campaign: Campaign) -> dict[str, Any]:
return {
"id": campaign.id,
"external_id": campaign.external_id,
"name": campaign.name,
"description": campaign.description,
"status": campaign.status,
"created_at": campaign.created_at.isoformat() if campaign.created_at else None,
"updated_at": campaign.updated_at.isoformat() if campaign.updated_at else None,
}
def _campaign_report_attempt_counts(session: Session, job_ids: list[str]) -> dict[str, int]:
return {
"send_attempts": int(session.query(SendAttempt).filter(SendAttempt.job_id.in_(job_ids)).count()) if job_ids else 0,
"imap_append_attempts": int(session.query(ImapAppendAttempt).filter(ImapAppendAttempt.job_id.in_(job_ids)).count()) if job_ids else 0,
}
def _persisted_campaign_issue_count(
session: Session,
*,
tenant_id: str,
campaign_id: str,
version: CampaignVersion | None,
) -> int:
issue_query = session.query(CampaignIssue).filter(
CampaignIssue.tenant_id == tenant_id,
CampaignIssue.campaign_id == campaign.id,
CampaignIssue.campaign_id == campaign_id,
)
if version:
issue_query = issue_query.filter(CampaignIssue.campaign_version_id == version.id)
else:
issue_query = issue_query.filter(False)
persisted_issues = issue_query.count()
return int(issue_query.count())
def _campaign_report_status_counts(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, dict[str, int]]:
validation_counts = _counter([job.validation_status for job in jobs])
queue_counts = _counter([job.queue_status for job in jobs])
inactive_entries = _inactive_entry_count(version)
if inactive_entries:
validation_counts["inactive"] = inactive_entries
return {
"build": _counter([job.build_status for job in jobs]),
"validation": validation_counts,
"queue": _counter([job.queue_status for job in jobs]),
"send": _counter([job.send_status for job in jobs]),
"imap": _counter([job.imap_status for job in jobs]),
}
def _campaign_report_cards(version: CampaignVersion | None, jobs: list[CampaignJob]) -> dict[str, object]:
send_counts = _counter([job.send_status for job in jobs])
imap_counts = _counter([job.imap_status for job in jobs])
build_counts = _counter([job.build_status for job in jobs])
queueable = sum(1 for job in jobs if job.validation_status in {"ready", "warning"} and job.build_status == "built")
needs_attention = sum(
1
@@ -413,63 +502,28 @@ def generate_campaign_report(
not_attempted = send_counts.get("not_queued", 0)
queued = send_counts.get("queued", 0) + send_counts.get("claimed", 0) + send_counts.get("sending", 0)
cancelled = send_counts.get("cancelled", 0)
build_summary = version.build_summary if version and isinstance(version.build_summary, dict) else {}
inactive_entries = int(build_summary.get("inactive_count") or build_summary.get("inactive_entries_count") or 0)
if inactive_entries:
validation_counts["inactive"] = inactive_entries
report: dict[str, Any] = {
"generated_at": _utcnow_iso(),
"campaign": {
"id": campaign.id,
"external_id": campaign.external_id,
"name": campaign.name,
"description": campaign.description,
"status": campaign.status,
"created_at": campaign.created_at.isoformat() if campaign.created_at else None,
"updated_at": campaign.updated_at.isoformat() if campaign.updated_at else None,
},
"current_version": _version_info(version),
"selected_version_id": version.id if version else None,
"cards": {
"jobs_total": len(jobs),
"inactive": inactive_entries,
"queueable": queueable,
"needs_attention": needs_attention,
"sent": sent,
"smtp_accepted": sent,
"failed": failed,
"outcome_unknown": outcome_unknown,
"not_attempted": not_attempted,
"queued_or_active": queued,
"cancelled": cancelled,
"partially_completed": bool(sent and (failed or outcome_unknown or not_attempted or cancelled)),
"imap_appended": imap_counts.get("appended", 0),
"imap_failed": imap_counts.get("failed", 0),
},
"status_counts": {
"build": build_counts,
"validation": validation_counts,
"queue": queue_counts,
"send": send_counts,
"imap": imap_counts,
},
"issues": {
**_issue_summary_from_jobs(jobs),
"persisted_campaign_issue_count": persisted_issues,
},
"attachments": _attachment_summary(jobs),
"attempts": {
"send_attempts": int(send_attempts),
"imap_append_attempts": int(imap_attempts),
},
"delivery": _load_delivery_info(version, jobs),
inactive_entries = _inactive_entry_count(version)
return {
"jobs_total": len(jobs),
"inactive": inactive_entries,
"queueable": queueable,
"needs_attention": needs_attention,
"sent": sent,
"smtp_accepted": sent,
"failed": failed,
"outcome_unknown": outcome_unknown,
"not_attempted": not_attempted,
"queued_or_active": queued,
"cancelled": cancelled,
"partially_completed": bool(sent and (failed or outcome_unknown or not_attempted or cancelled)),
"imap_appended": imap_counts.get("appended", 0),
"imap_failed": imap_counts.get("failed", 0),
}
if include_recent_failures:
report["recent_failures"] = _recent_failures(jobs)
if include_jobs:
report["jobs"] = [_job_row(job) for job in jobs]
return report
def _inactive_entry_count(version: CampaignVersion | None) -> int:
build_summary = version.build_summary if version and isinstance(version.build_summary, dict) else {}
return int(build_summary.get("inactive_count") or build_summary.get("inactive_entries_count") or 0)
def generate_jobs_csv(

View File

@@ -105,7 +105,7 @@ def _text_summary(report: dict[str, Any]) -> str:
]
if delivery.get("estimated_remaining_send_human"):
lines.extend(["", f"Estimated remaining send time: {delivery['estimated_remaining_send_human']}"])
lines.extend(["", "This report was generated by MultiMailer."])
lines.extend(["", "This report was generated by GovOPlaN."])
return "\n".join(lines)
@@ -119,20 +119,20 @@ def build_report_message(
report_json: dict[str, Any] | None = None,
) -> EmailMessage:
from_email, from_name = _effective_from(config)
subject = f"MultiMailer report: {campaign.name}"
subject = f"GovOPlaN report: {campaign.name}"
msg = EmailMessage()
msg["Subject"] = subject
msg["From"] = formataddr((from_name or from_email, from_email))
msg["To"] = ", ".join(to)
msg["X-MultiMailer-Report"] = "campaign"
msg["X-GovOPlaN-Report"] = "campaign"
msg.set_content(_text_summary(report))
if jobs_csv is not None:
filename = f"multimailer-{campaign.external_id}-jobs.csv"
filename = f"govoplan-{campaign.external_id}-jobs.csv"
msg.add_attachment(jobs_csv.encode("utf-8"), maintype="text", subtype="csv", filename=filename)
if report_json is not None:
filename = f"multimailer-{campaign.external_id}-report.json"
filename = f"govoplan-{campaign.external_id}-report.json"
msg.add_attachment(
json.dumps(report_json, indent=2, ensure_ascii=False, default=str).encode("utf-8"),
maintype="application",

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,7 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://multimailer.local/schema/campaign.schema.json",
"title": "MultiMailer Campaign",
"$id": "https://govoplan.local/schema/campaign.schema.json",
"title": "GovOPlaN Campaign",
"type": "object",
"required": [
"version",
@@ -1074,9 +1074,32 @@
"enum": [
"csv",
"xlsx",
"text"
"text",
"addresses"
]
},
"source_id": {
"type": [
"string",
"null"
]
},
"source_label": {
"type": [
"string",
"null"
]
},
"source_revision": {
"type": [
"string",
"null"
]
},
"source_provenance": {
"type": "object",
"default": {}
},
"filename": {
"type": [
"string",

View File

@@ -266,6 +266,65 @@ class RecipientImportMappingProfileListResponse(BaseModel):
profiles: list[RecipientImportMappingProfileResponse] = Field(default_factory=list)
class CampaignAddressLookupCandidate(BaseModel):
contact_id: str
address_book_id: str
display_name: str
email: str | None = None
email_label: str | None = None
organization: str | None = None
role_title: str | None = None
tags: list[str] = Field(default_factory=list)
source_kind: str = "local"
source_ref: str | None = None
source_revision: str | None = None
provenance: dict[str, Any] = Field(default_factory=dict)
class CampaignAddressLookupResponse(BaseModel):
available: bool = False
candidates: list[CampaignAddressLookupCandidate] = Field(default_factory=list)
class CampaignRecipientAddressSource(BaseModel):
source_id: str
source_label: str
source_kind: str
source_revision: str
recipient_count: int = 0
provenance: dict[str, Any] = Field(default_factory=dict)
class CampaignRecipientAddressSourcesResponse(BaseModel):
available: bool = False
sources: list[CampaignRecipientAddressSource] = Field(default_factory=list)
class CampaignRecipientAddressSourceSnapshotRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
source_id: str = Field(min_length=1)
class CampaignRecipientSnapshotItem(BaseModel):
contact_id: str
display_name: str
email: str
email_label: str | None = None
fields: dict[str, Any] = Field(default_factory=dict)
provenance: dict[str, Any] = Field(default_factory=dict)
class CampaignRecipientAddressSourceSnapshotResponse(BaseModel):
source_id: str
source_label: str
source_kind: str
source_revision: str
generated_at: str
recipients: list[CampaignRecipientSnapshotItem] = Field(default_factory=list)
provenance: dict[str, Any] = Field(default_factory=dict)
class CampaignJobsResponse(BaseModel):
jobs: list[dict[str, Any]]
page: int = 1

View File

@@ -219,8 +219,10 @@ def create_execution_snapshot(
queueable_statuses = {JobValidationStatus.READY.value, JobValidationStatus.WARNING.value}
redacted_smtp = _redacted_transport_config(smtp)
redacted_imap = _redacted_transport_config(imap)
assert isinstance(redacted_smtp, SmtpConfig)
assert redacted_imap is None or isinstance(redacted_imap, ImapConfig)
if not isinstance(redacted_smtp, SmtpConfig):
raise ExecutionSnapshotError("Redacted SMTP configuration is invalid.")
if redacted_imap is not None and not isinstance(redacted_imap, ImapConfig):
raise ExecutionSnapshotError("Redacted IMAP configuration is invalid.")
payload = ExecutionSnapshot(
campaign_version_id=version.id,
campaign_json_sha256=_sha256(raw_json),

View File

@@ -12,6 +12,7 @@ from uuid import uuid4
from sqlalchemy.orm import Session
from govoplan_core.core.notifications import NotificationDispatchRequest, notification_dispatch_provider
from govoplan_core.settings import settings as core_settings
from govoplan_campaign.backend.db.models import (
Campaign,
@@ -28,6 +29,7 @@ from govoplan_campaign.backend.db.models import (
SendAttempt,
)
from govoplan_campaign.backend.sending.execution import ExecutionSnapshot, ExecutionSnapshotError, ensure_execution_snapshot, runtime_imap_config, runtime_smtp_config
from govoplan_campaign.backend.runtime import get_registry
from govoplan_campaign.backend.integrations import (
ImapAppendError,
ImapConfigurationError,
@@ -133,6 +135,25 @@ class AppendSentResult:
}
@dataclass(frozen=True, slots=True)
class _CampaignDeliveryCounts:
accepted: int
unknown: int
failed: int
active: int
cancelled: int
not_started: int
@dataclass(frozen=True, slots=True)
class _SendJobDeliveryContext:
version: CampaignVersion
snapshot: ExecutionSnapshot
message_bytes: bytes
envelope_from: str
envelope_recipients: list[str]
QUEUEABLE_VALIDATION_STATUSES = {
JobValidationStatus.READY.value,
JobValidationStatus.WARNING.value,
@@ -140,6 +161,15 @@ QUEUEABLE_VALIDATION_STATUSES = {
SMTP_ACCEPTED_STATUSES = {JobSendStatus.SMTP_ACCEPTED.value, JobSendStatus.SENT.value}
AUTOMATICALLY_SENDABLE_STATUSES = {JobSendStatus.QUEUED.value}
EXPLICIT_RETRY_STATUSES = {JobSendStatus.FAILED_TEMPORARY.value, JobSendStatus.FAILED_PERMANENT.value}
CAMPAIGN_STATUS_NOTIFICATION_EVENTS = {
CampaignStatus.QUEUED.value: ("campaign.queued", "Campaign queued", 1),
CampaignStatus.SENDING.value: ("campaign.sending", "Campaign sending started", 1),
CampaignStatus.SENT.value: ("campaign.sent", "Campaign sent", 1),
CampaignStatus.PARTIALLY_COMPLETED.value: ("campaign.partially_completed", "Campaign partially completed", 4),
CampaignStatus.OUTCOME_UNKNOWN.value: ("campaign.outcome_unknown", "Campaign requires review", 5),
CampaignStatus.FAILED.value: ("campaign.failed", "Campaign failed", 5),
CampaignStatus.CANCELLED.value: ("campaign.cancelled", "Campaign cancelled", 2),
}
def _version_user_lock_state(version: CampaignVersion) -> str | None:
@@ -210,16 +240,72 @@ def _should_enqueue_celery(enqueue_celery: bool) -> bool:
return bool(enqueue_celery and _celery_enabled())
def _campaign_notification_body(campaign: Campaign, status: str) -> str:
return {
CampaignStatus.QUEUED.value: f"{campaign.name} has been queued for delivery.",
CampaignStatus.SENDING.value: f"{campaign.name} started sending.",
CampaignStatus.SENT.value: f"{campaign.name} finished sending.",
CampaignStatus.PARTIALLY_COMPLETED.value: f"{campaign.name} finished with partial delivery results. Review the campaign report.",
CampaignStatus.OUTCOME_UNKNOWN.value: f"{campaign.name} has unresolved delivery outcomes and needs operator review.",
CampaignStatus.FAILED.value: f"{campaign.name} failed during delivery. Review the failed jobs.",
CampaignStatus.CANCELLED.value: f"{campaign.name} was cancelled.",
}.get(status, f"{campaign.name} changed status to {status}.")
def _emit_campaign_status_notification(
session: Session,
*,
campaign: Campaign,
status: str,
previous_status: str | None = None,
version_id: str | None = None,
message: str | None = None,
) -> None:
provider = notification_dispatch_provider(get_registry())
if provider is None:
return
event = CAMPAIGN_STATUS_NOTIFICATION_EVENTS.get(status)
if event is None:
return
event_kind, title, priority = event
try:
provider.enqueue_notification(
session,
NotificationDispatchRequest(
tenant_id=campaign.tenant_id,
source_module="campaigns",
source_resource_type="campaign",
source_resource_id=campaign.id,
event_kind=event_kind,
channel="inbox",
subject=f"{title}: {campaign.name}",
body_text=message or _campaign_notification_body(campaign, status),
action_url=f"/campaigns/{campaign.id}",
priority=priority,
payload={
"campaign_id": campaign.id,
"version_id": version_id or campaign.current_version_id,
"status": status,
"previous_status": previous_status,
},
metadata={"campaign_id": campaign.id, "version_id": version_id or campaign.current_version_id},
),
enqueue_delivery=False,
)
except Exception:
return
def _celery_enqueue_send_job(job_id: str) -> None:
from govoplan_core.celery_app import celery
celery.send_task("multimailer.send_email", args=[job_id], queue="send_email")
celery.send_task("govoplan.campaigns.send_email", args=[job_id], queue="send_email")
def _celery_enqueue_append_sent_job(job_id: str) -> None:
from govoplan_core.celery_app import celery
celery.send_task("multimailer.append_sent", args=[job_id], queue="append_sent")
celery.send_task("govoplan.campaigns.append_sent", args=[job_id], queue="append_sent")
def queue_campaign_jobs(
@@ -296,11 +382,20 @@ def queue_campaign_jobs(
if not dry_run:
if queued:
previous_status = campaign.status
campaign.status = CampaignStatus.QUEUED.value
version.workflow_state = CampaignVersionWorkflowState.QUEUED.value
if version.locked_at is None:
version.locked_at = _utcnow()
session.add(version)
if previous_status != campaign.status:
_emit_campaign_status_notification(
session,
campaign=campaign,
status=campaign.status,
previous_status=previous_status,
version_id=version.id,
)
session.add(campaign)
session.commit()
@@ -469,8 +564,16 @@ def resume_campaign_jobs(session: Session, *, tenant_id: str, campaign_id: str,
job.send_status = JobSendStatus.QUEUED.value
session.add(job)
if jobs:
previous_status = campaign.status
campaign.status = CampaignStatus.QUEUED.value
session.add(campaign)
if previous_status != campaign.status:
_emit_campaign_status_notification(
session,
campaign=campaign,
status=campaign.status,
previous_status=previous_status,
)
session.commit()
enqueued_count = 0
@@ -909,27 +1012,54 @@ def _update_campaign_after_job(session: Session, campaign_id: str, version_id: s
campaign = session.get(Campaign, campaign_id)
if not campaign:
return
base_filters = [CampaignJob.campaign_id == campaign_id]
if version_id:
base_filters.append(CampaignJob.campaign_version_id == version_id)
previous_status = campaign.status
version = session.get(CampaignVersion, version_id) if version_id else None
counts = _campaign_delivery_counts(session, campaign_id=campaign_id, version_id=version_id)
_apply_campaign_delivery_status(campaign, version, counts)
if version and (counts.accepted or counts.unknown):
if version.locked_at is None:
version.locked_at = _utcnow()
session.add(version)
session.add(campaign)
if campaign.status != previous_status:
_emit_campaign_status_notification(
session,
campaign=campaign,
status=campaign.status,
previous_status=previous_status,
version_id=version_id,
)
def _campaign_delivery_counts(session: Session, *, campaign_id: str, version_id: str | None) -> _CampaignDeliveryCounts:
base_filters = _campaign_delivery_base_filters(campaign_id=campaign_id, version_id=version_id)
counts = {
status: session.query(CampaignJob).filter(*base_filters, CampaignJob.send_status == status).count()
for status in {item.value for item in JobSendStatus}
}
accepted = sum(counts.get(status, 0) for status in SMTP_ACCEPTED_STATUSES)
unknown = counts.get(JobSendStatus.OUTCOME_UNKNOWN.value, 0)
failed = counts.get(JobSendStatus.FAILED_TEMPORARY.value, 0) + counts.get(JobSendStatus.FAILED_PERMANENT.value, 0)
active = (
counts.get(JobSendStatus.QUEUED.value, 0)
+ counts.get(JobSendStatus.CLAIMED.value, 0)
+ counts.get(JobSendStatus.SENDING.value, 0)
return _CampaignDeliveryCounts(
accepted=sum(counts.get(status, 0) for status in SMTP_ACCEPTED_STATUSES),
unknown=counts.get(JobSendStatus.OUTCOME_UNKNOWN.value, 0),
failed=counts.get(JobSendStatus.FAILED_TEMPORARY.value, 0) + counts.get(JobSendStatus.FAILED_PERMANENT.value, 0),
active=(
counts.get(JobSendStatus.QUEUED.value, 0)
+ counts.get(JobSendStatus.CLAIMED.value, 0)
+ counts.get(JobSendStatus.SENDING.value, 0)
),
cancelled=counts.get(JobSendStatus.CANCELLED.value, 0),
not_started=_queueable_not_started_job_count(session, base_filters),
)
cancelled = counts.get(JobSendStatus.CANCELLED.value, 0)
# Blocked/excluded build rows are reportable but are not delivery attempts.
# Only a built, queueable message counts as an unattempted part of an
# execution when deriving a partial outcome.
not_started = (
def _campaign_delivery_base_filters(*, campaign_id: str, version_id: str | None) -> list[object]:
base_filters: list[object] = [CampaignJob.campaign_id == campaign_id]
if version_id:
base_filters.append(CampaignJob.campaign_version_id == version_id)
return base_filters
def _queueable_not_started_job_count(session: Session, base_filters: list[object]) -> int:
return (
session.query(CampaignJob)
.filter(
*base_filters,
@@ -940,37 +1070,35 @@ def _update_campaign_after_job(session: Session, campaign_id: str, version_id: s
.count()
)
version = session.get(CampaignVersion, version_id) if version_id else None
if active:
campaign.status = CampaignStatus.SENDING.value
if version:
version.workflow_state = CampaignVersionWorkflowState.SENDING.value
elif accepted and (failed or unknown or cancelled or not_started):
campaign.status = CampaignStatus.PARTIALLY_COMPLETED.value
if version:
version.workflow_state = CampaignVersionWorkflowState.PARTIALLY_COMPLETED.value
elif unknown:
campaign.status = CampaignStatus.OUTCOME_UNKNOWN.value
if version:
version.workflow_state = CampaignVersionWorkflowState.OUTCOME_UNKNOWN.value
elif failed:
campaign.status = CampaignStatus.FAILED.value
if version:
version.workflow_state = CampaignVersionWorkflowState.FAILED.value
elif accepted:
campaign.status = CampaignStatus.SENT.value
if version:
version.workflow_state = CampaignVersionWorkflowState.COMPLETED.value
elif cancelled:
campaign.status = CampaignStatus.CANCELLED.value
if version:
version.workflow_state = CampaignVersionWorkflowState.CANCELLED.value
if version and (accepted or unknown):
if version.locked_at is None:
version.locked_at = _utcnow()
session.add(version)
session.add(campaign)
def _apply_campaign_delivery_status(
campaign: Campaign,
version: CampaignVersion | None,
counts: _CampaignDeliveryCounts,
) -> None:
status_pair = _campaign_delivery_status_pair(counts)
if status_pair is None:
return
campaign_status, workflow_state = status_pair
campaign.status = campaign_status
if version:
version.workflow_state = workflow_state
def _campaign_delivery_status_pair(counts: _CampaignDeliveryCounts) -> tuple[str, str] | None:
if counts.active:
return CampaignStatus.SENDING.value, CampaignVersionWorkflowState.SENDING.value
if counts.accepted and (counts.failed or counts.unknown or counts.cancelled or counts.not_started):
return CampaignStatus.PARTIALLY_COMPLETED.value, CampaignVersionWorkflowState.PARTIALLY_COMPLETED.value
if counts.unknown:
return CampaignStatus.OUTCOME_UNKNOWN.value, CampaignVersionWorkflowState.OUTCOME_UNKNOWN.value
if counts.failed:
return CampaignStatus.FAILED.value, CampaignVersionWorkflowState.FAILED.value
if counts.accepted:
return CampaignStatus.SENT.value, CampaignVersionWorkflowState.COMPLETED.value
if counts.cancelled:
return CampaignStatus.CANCELLED.value, CampaignVersionWorkflowState.CANCELLED.value
return None
def send_campaign_job(
@@ -984,15 +1112,49 @@ def send_campaign_job(
job = session.get(CampaignJob, job_id)
if not job:
raise SendJobError(f"Job not found: {job_id}")
preflight_result = _preflight_send_campaign_job(session, job, dry_run=dry_run)
if preflight_result is not None:
return preflight_result
context = _send_job_delivery_context(session, job)
if dry_run:
return SendJobResult(
job_id=job.id,
status="dry_run",
attempt_number=job.attempt_count,
dry_run=True,
message=f"Would send to {len(context.envelope_recipients)} recipient(s) from {context.envelope_from}",
)
claimed = _claimed_campaign_job_for_delivery(session, job)
if isinstance(claimed, SendJobResult):
return claimed
claimed_job, claim_token = claimed
return _send_claimed_campaign_job(
session,
job=claimed_job,
claim_token=claim_token,
context=context,
use_rate_limit=use_rate_limit,
enqueue_imap_task=enqueue_imap_task,
)
def _preflight_send_campaign_job(
session: Session,
job: CampaignJob,
*,
dry_run: bool,
) -> SendJobResult | None:
if job.queue_status == JobQueueStatus.CANCELLED.value or job.send_status == JobSendStatus.CANCELLED.value:
return SendJobResult(job_id=job_id, status="cancelled", attempt_number=job.attempt_count, dry_run=dry_run)
return SendJobResult(job_id=job.id, status="cancelled", attempt_number=job.attempt_count, dry_run=dry_run)
if job.queue_status == JobQueueStatus.PAUSED.value:
return SendJobResult(job_id=job_id, status="paused", attempt_number=job.attempt_count, dry_run=dry_run)
return SendJobResult(job_id=job.id, status="paused", attempt_number=job.attempt_count, dry_run=dry_run)
if job.send_status in SMTP_ACCEPTED_STATUSES:
return SendJobResult(job_id=job_id, status="already_accepted", attempt_number=job.attempt_count, dry_run=dry_run)
return SendJobResult(job_id=job.id, status="already_accepted", attempt_number=job.attempt_count, dry_run=dry_run)
if job.send_status == JobSendStatus.OUTCOME_UNKNOWN.value:
return SendJobResult(
job_id=job_id,
job_id=job.id,
status=JobSendStatus.OUTCOME_UNKNOWN.value,
attempt_number=job.attempt_count,
dry_run=dry_run,
@@ -1014,7 +1176,10 @@ def send_campaign_job(
)
if job.queue_status != JobQueueStatus.QUEUED.value or job.send_status != JobSendStatus.QUEUED.value:
raise SendJobError(f"Job is not explicitly queued for delivery: queue={job.queue_status}, send={job.send_status}")
return None
def _send_job_delivery_context(session: Session, job: CampaignJob) -> _SendJobDeliveryContext:
version = session.get(CampaignVersion, job.campaign_version_id)
if not version:
raise SendJobError("Campaign version not found")
@@ -1028,134 +1193,196 @@ def send_campaign_job(
envelope_recipients = _recipients_from_job(job)
if not envelope_recipients:
raise SmtpConfigurationError("No envelope recipients could be determined")
return _SendJobDeliveryContext(
version=version,
snapshot=snapshot,
message_bytes=message_bytes,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
)
if dry_run:
return SendJobResult(
job_id=job.id,
status="dry_run",
attempt_number=job.attempt_count,
dry_run=True,
message=f"Would send to {len(envelope_recipients)} recipient(s) from {envelope_from}",
)
def _claimed_campaign_job_for_delivery(
session: Session,
job: CampaignJob,
) -> tuple[CampaignJob, str] | SendJobResult:
claim_token = _claim_job_for_sending(session, job)
if claim_token is None:
current = session.get(CampaignJob, job.id)
if not current:
raise SendJobError(f"Job disappeared while claiming: {job.id}")
if current.send_status == JobSendStatus.SENDING.value:
return mark_job_outcome_unknown(
session,
current,
reason="A duplicate/redelivered task found an unfinished SMTP attempt. Automatic resend was stopped.",
)
return SendJobResult(
job_id=current.id,
status="not_claimed",
attempt_number=current.attempt_count,
message=f"Job is no longer queueable: {current.send_status}",
)
return _not_claimed_send_job_result(session, job)
job = session.get(CampaignJob, job.id)
assert job is not None
if job is None:
raise SendJobError("Claimed campaign job disappeared before send.")
return job, claim_token
def _not_claimed_send_job_result(session: Session, job: CampaignJob) -> SendJobResult:
current = session.get(CampaignJob, job.id)
if not current:
raise SendJobError(f"Job disappeared while claiming: {job.id}")
if current.send_status == JobSendStatus.SENDING.value:
return mark_job_outcome_unknown(
session,
current,
reason="A duplicate/redelivered task found an unfinished SMTP attempt. Automatic resend was stopped.",
)
return SendJobResult(
job_id=current.id,
status="not_claimed",
attempt_number=current.attempt_count,
message=f"Job is no longer queueable: {current.send_status}",
)
def _send_claimed_campaign_job(
session: Session,
*,
job: CampaignJob,
claim_token: str,
context: _SendJobDeliveryContext,
use_rate_limit: bool,
enqueue_imap_task: bool,
) -> SendJobResult:
mail_integration().wait_for_rate_limit(
key=f"tenant:{job.tenant_id}:campaign:{job.campaign_id}",
messages_per_minute=snapshot.delivery.rate_limit.messages_per_minute,
messages_per_minute=context.snapshot.delivery.rate_limit.messages_per_minute,
enabled=use_rate_limit,
)
attempt = _record_attempt_start(session, job, claim_token)
try:
smtp_config = runtime_smtp_config(session, version, snapshot)
smtp_config = runtime_smtp_config(session, context.version, context.snapshot)
mail_integration().assert_mail_policy_allows_send(
session,
tenant_id=job.tenant_id,
campaign_id=job.campaign_id,
smtp=smtp_config,
envelope_sender=envelope_from,
from_header=_from_header_from_job(job, snapshot),
recipients=envelope_recipients,
envelope_sender=context.envelope_from,
from_header=_from_header_from_job(job, context.snapshot),
recipients=context.envelope_recipients,
)
result = mail_integration().send_email_bytes(
message_bytes,
context.message_bytes,
smtp_config=smtp_config,
envelope_from=envelope_from,
envelope_recipients=envelope_recipients,
envelope_from=context.envelope_from,
envelope_recipients=context.envelope_recipients,
)
if result.accepted_count <= 0:
raise SmtpSendError("SMTP did not accept any envelope recipients", temporary=False)
refused_warning = None
if result.refused_recipients:
refused_warning = (
f"SMTP accepted {result.accepted_count}/{len(result.envelope_recipients)} envelope recipient(s); "
f"refused recipients: {json.dumps(result.refused_recipients, default=str, sort_keys=True)}"
)
attempt.finished_at = _utcnow()
attempt.status = "smtp_accepted_with_refusals" if refused_warning else JobSendStatus.SMTP_ACCEPTED.value
attempt.smtp_response = json.dumps(asdict(result), default=str)
job.queue_status = JobQueueStatus.DRAFT.value
job.send_status = JobSendStatus.SMTP_ACCEPTED.value
job.sent_at = _utcnow()
job.claim_token = None
job.outcome_unknown_at = None
if snapshot.delivery.imap_append_sent.enabled:
job.imap_status = JobImapStatus.PENDING.value
else:
job.imap_status = JobImapStatus.NOT_REQUESTED.value
job.last_error = refused_warning
files_integration().mark_job_attachment_uses_sent(session, job)
session.add(attempt)
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
if enqueue_imap_task and _celery_enabled() and job.imap_status == JobImapStatus.PENDING.value:
_celery_enqueue_append_sent_job(job.id)
return SendJobResult(
job_id=job.id,
status=JobSendStatus.SMTP_ACCEPTED.value,
attempt_number=attempt.attempt_number,
message=refused_warning,
return _record_smtp_send_success(
session,
job=job,
attempt=attempt,
snapshot=context.snapshot,
result=result,
enqueue_imap_task=enqueue_imap_task,
)
except SmtpSendError as exc:
if getattr(exc, "outcome_unknown", False):
attempt.status = JobSendStatus.OUTCOME_UNKNOWN.value
attempt.finished_at = _utcnow()
attempt.error_type = exc.__class__.__name__
attempt.error_message = str(exc)
session.add(attempt)
job.claim_token = None
session.add(job)
session.flush()
return mark_job_outcome_unknown(session, job, reason=str(exc))
attempt.finished_at = _utcnow()
attempt.error_type = exc.__class__.__name__
attempt.error_message = str(exc)
retryable = bool(getattr(exc, "temporary", False))
job.last_error = str(exc)
job.queue_status = JobQueueStatus.DRAFT.value
job.send_status = JobSendStatus.FAILED_TEMPORARY.value if retryable else JobSendStatus.FAILED_PERMANENT.value
job.claim_token = None
attempt.status = job.send_status
session.add(attempt)
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
outcome_unknown = _record_smtp_send_error(session, job=job, attempt=attempt, exc=exc)
if outcome_unknown is not None:
return outcome_unknown
raise
except (MailProfileError, SmtpConfigurationError, SendJobError, OSError) as exc:
_record_permanent_send_error(session, job=job, attempt=attempt, exc=exc)
raise
def _record_smtp_send_success(
session: Session,
*,
job: CampaignJob,
attempt: SendAttempt,
snapshot: ExecutionSnapshot,
result: object,
enqueue_imap_task: bool,
) -> SendJobResult:
if result.accepted_count <= 0:
raise SmtpSendError("SMTP did not accept any envelope recipients", temporary=False)
refused_warning = _refused_recipient_warning(result)
attempt.finished_at = _utcnow()
attempt.status = "smtp_accepted_with_refusals" if refused_warning else JobSendStatus.SMTP_ACCEPTED.value
attempt.smtp_response = json.dumps(asdict(result), default=str)
job.queue_status = JobQueueStatus.DRAFT.value
job.send_status = JobSendStatus.SMTP_ACCEPTED.value
job.sent_at = _utcnow()
job.claim_token = None
job.outcome_unknown_at = None
job.imap_status = JobImapStatus.PENDING.value if snapshot.delivery.imap_append_sent.enabled else JobImapStatus.NOT_REQUESTED.value
job.last_error = refused_warning
files_integration().mark_job_attachment_uses_sent(session, job)
session.add(attempt)
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
if enqueue_imap_task and _celery_enabled() and job.imap_status == JobImapStatus.PENDING.value:
_celery_enqueue_append_sent_job(job.id)
return SendJobResult(
job_id=job.id,
status=JobSendStatus.SMTP_ACCEPTED.value,
attempt_number=attempt.attempt_number,
message=refused_warning,
)
def _refused_recipient_warning(result: object) -> str | None:
if not result.refused_recipients:
return None
return (
f"SMTP accepted {result.accepted_count}/{len(result.envelope_recipients)} envelope recipient(s); "
f"refused recipients: {json.dumps(result.refused_recipients, default=str, sort_keys=True)}"
)
def _record_smtp_send_error(
session: Session,
*,
job: CampaignJob,
attempt: SendAttempt,
exc: SmtpSendError,
) -> SendJobResult | None:
if getattr(exc, "outcome_unknown", False):
attempt.status = JobSendStatus.OUTCOME_UNKNOWN.value
attempt.finished_at = _utcnow()
attempt.error_type = exc.__class__.__name__
attempt.error_message = str(exc)
attempt.status = JobSendStatus.FAILED_PERMANENT.value
job.last_error = str(exc)
job.queue_status = JobQueueStatus.DRAFT.value
job.send_status = JobSendStatus.FAILED_PERMANENT.value
job.claim_token = None
session.add(attempt)
job.claim_token = None
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
raise
session.flush()
return mark_job_outcome_unknown(session, job, reason=str(exc))
attempt.finished_at = _utcnow()
attempt.error_type = exc.__class__.__name__
attempt.error_message = str(exc)
retryable = bool(getattr(exc, "temporary", False))
job.last_error = str(exc)
job.queue_status = JobQueueStatus.DRAFT.value
job.send_status = JobSendStatus.FAILED_TEMPORARY.value if retryable else JobSendStatus.FAILED_PERMANENT.value
job.claim_token = None
attempt.status = job.send_status
session.add(attempt)
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
return None
def _record_permanent_send_error(
session: Session,
*,
job: CampaignJob,
attempt: SendAttempt,
exc: Exception,
) -> None:
attempt.finished_at = _utcnow()
attempt.error_type = exc.__class__.__name__
attempt.error_message = str(exc)
attempt.status = JobSendStatus.FAILED_PERMANENT.value
job.last_error = str(exc)
job.queue_status = JobQueueStatus.DRAFT.value
job.send_status = JobSendStatus.FAILED_PERMANENT.value
job.claim_token = None
session.add(attempt)
session.add(job)
_update_campaign_after_job(session, job.campaign_id, job.campaign_version_id)
session.commit()
def _record_imap_attempt_start(session: Session, job: CampaignJob) -> ImapAppendAttempt: