from __future__ import annotations import mimetypes import re import tempfile import time from dataclasses import dataclass from email.message import EmailMessage from email.utils import make_msgid, formatdate from pathlib import Path from typing import Any, Iterable from govoplan_campaign.backend.attachments.resolver import ( AttachmentMatchIndex, AttachmentMatchStatus, EntryAttachmentResolution, MessageAttachmentStatus, ResolvedAttachment, resolve_entry_attachments, ) from govoplan_campaign.backend.campaign.addressing import effective_address_lists, formatted_recipient from govoplan_campaign.backend.campaign.entries import load_campaign_entries from govoplan_campaign.backend.campaign.field_values import ignored_entry_field_overrides from govoplan_campaign.backend.campaign.models import ( Behavior, BuildStatus, CampaignConfig, EntryConfig, MissingAddressBehavior, RecipientConfig, SendStatus, TemplateBodyMode, ZipArchiveConfig, ZipPasswordMode, ZipPasswordScope, ) from govoplan_campaign.backend.campaign.template_values import build_template_values from govoplan_campaign.backend.services.zip_service import create_zip_archive from .models import ( CampaignBuildReport, ImapStatus, MessageAddress, MessageAttachmentSummary, MessageDraft, MessageIssue, MessageValidationStatus, ) _DOLLAR_FIELD_PATTERN = re.compile(r"(? str: key = raw.strip() if key.startswith("fields."): key = key.removeprefix("fields.") elif key.startswith("local."): key = "local::" + key.removeprefix("local.") elif key.startswith("global."): key = "global::" + key.removeprefix("global.") if key.startswith("local::") or key.startswith("global::"): return key if key.startswith("local:"): return "local::" + key.removeprefix("local:") if key.startswith("global:"): return "global::" + key.removeprefix("global:") return key @dataclass(slots=True) class BuiltMessage: draft: MessageDraft mime: EmailMessage | None @dataclass(slots=True) class CampaignBuildResult: report: CampaignBuildReport built_messages: list[BuiltMessage] def _resolve(campaign_file: str | Path, raw_path: str) -> Path: campaign_path = Path(campaign_file).resolve() path = Path(raw_path).expanduser() if path.is_absolute(): return path return (campaign_path.parent / path).resolve() def _read_text(campaign_file: str | Path, raw_path: str | None, encoding: str = "utf-8") -> str | None: if not raw_path: return None path = _resolve(campaign_file, raw_path) return path.read_text(encoding=encoding) def _render_template(template: str, values: dict[str, Any], *, keep_missing: bool = True) -> str: def replace(match: re.Match[str]) -> str: key = _normalize_template_key(match.group(1)) if key in values: value = values[key] return "" if value is None else str(value) return match.group(0) if keep_missing else "" rendered = _DOLLAR_FIELD_PATTERN.sub(replace, template) rendered = _BRACE_FIELD_PATTERN.sub(replace, rendered) return rendered.replace(r"\${", "${").replace(r"\}", "}") def _find_unresolved_placeholders(text: str | None) -> set[str]: if not text: return set() return { _normalize_template_key(match.group(1)) for pattern in (_DOLLAR_FIELD_PATTERN, _BRACE_FIELD_PATTERN) for match in pattern.finditer(text) } def _message_address(recipient: RecipientConfig | None) -> MessageAddress | None: if recipient is None: return None return MessageAddress(email=recipient.email, name=recipient.name) def _message_addresses(recipients: Iterable[RecipientConfig]) -> list[MessageAddress]: return [MessageAddress(email=recipient.email, name=recipient.name) for recipient in recipients] def _format_recipient(recipient: RecipientConfig) -> str: return formatted_recipient(recipient) def _format_recipient_header(recipients: Iterable[RecipientConfig]) -> str: return ", ".join(_format_recipient(recipient) for recipient in recipients) def _load_template_parts(config: CampaignConfig, campaign_file: str | Path) -> tuple[str, str | None, str | None]: template = config.template if template.source: subject = _read_text(campaign_file, template.source.subject_path, template.source.encoding) text = _read_text(campaign_file, template.source.text_path, template.source.encoding) html = _read_text(campaign_file, template.source.html_path, template.source.encoding) return subject or "", text, html return template.subject or "", template.text, template.html def _template_body_mode(config: CampaignConfig) -> str: mode = config.template.body_mode if isinstance(mode, TemplateBodyMode): return mode.value return str(mode or TemplateBodyMode.BOTH.value) def _issue_from_behavior(*, code: str, message: str, behavior: str, source: str) -> MessageIssue: severity = "error" if behavior == "block" else "warning" return MessageIssue(severity=severity, code=code, message=message, behavior=behavior, source=source) def _apply_behavior(current: MessageValidationStatus, behavior: str) -> MessageValidationStatus: if behavior == Behavior.BLOCK.value: return MessageValidationStatus.BLOCKED if behavior == Behavior.DROP.value: return MessageValidationStatus.EXCLUDED if behavior == Behavior.ASK.value: if current not in {MessageValidationStatus.BLOCKED, MessageValidationStatus.EXCLUDED}: return MessageValidationStatus.NEEDS_REVIEW if behavior == Behavior.WARN.value: if current == MessageValidationStatus.READY: return MessageValidationStatus.WARNING # continue leaves status as-is return current def _validation_status_from_attachment_status(status: MessageAttachmentStatus) -> MessageValidationStatus: return MessageValidationStatus(status.value) def _attachment_summaries(resolution: EntryAttachmentResolution) -> list[MessageAttachmentSummary]: return [ MessageAttachmentSummary( attachment_id=attachment.attachment_id, label=attachment.label, status=attachment.status.value, behavior=attachment.behavior.value if attachment.behavior else None, required=attachment.required, allow_multiple=attachment.allow_multiple, zip_enabled=attachment.zip_enabled, zip_mode=attachment.zip_mode.value, zip_archive_id=attachment.zip_archive_id, zip_filename=attachment.zip_filename, message_filename_template=attachment.message_filename_template, zip_entry_name_template=attachment.zip_entry_name_template, message_filenames=attachment.message_filenames, zip_entry_names=attachment.zip_entry_names, base_path_name=attachment.base_path_name, base_path=attachment.base_path, file_filter=attachment.file_filter, directory=attachment.directory, matches=attachment.matches, ) for attachment in resolution.attachments ] def _message_issues_from_attachment_resolution(resolution: EntryAttachmentResolution) -> list[MessageIssue]: return [ MessageIssue( severity=issue.severity.value, code=issue.code, message=issue.message, behavior=issue.behavior.value if issue.behavior else None, source="attachments", ) for issue in resolution.issues ] def _safe_filename(value: str | None, fallback: str) -> str: raw = value or fallback safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", raw).strip("._") return safe or fallback def _attachment_bytes(path: Path) -> tuple[bytes, str, str]: data = path.read_bytes() mime_type, _ = mimetypes.guess_type(str(path)) if not mime_type: return data, "application", "octet-stream" maintype, subtype = mime_type.split("/", 1) return data, maintype, subtype class ZipBuildError(RuntimeError): def __init__(self, code: str, message: str): super().__init__(message) self.code = code def _zip_password(archive: ZipArchiveConfig, values: dict[str, Any]) -> str: if not archive.password_enabled: return "" # Preserve support for campaigns written by the original single-archive # implementation while all new archives use field + scope. mode = archive.password_mode if mode == ZipPasswordMode.DIRECT: password = archive.password or "" if not password: raise ZipBuildError("zip_password_missing", f"ZIP archive {archive.name!r} has no fixed password") return password if mode == ZipPasswordMode.TEMPLATE: password = _render_template(archive.password_template or "", values, keep_missing=False) if not password: raise ZipBuildError("zip_password_value_missing", f"ZIP archive {archive.name!r} produced an empty password") return password field_name = (archive.password_field or "").strip() if not field_name: raise ZipBuildError("zip_password_field_missing", f"ZIP archive {archive.name!r} has password protection enabled, but no password field") scope = archive.password_scope.value if isinstance(archive.password_scope, ZipPasswordScope) else str(archive.password_scope) value = values.get(f"{scope}::{field_name}") # Legacy field-based configurations did not declare a scope. if value in (None, "") and mode == ZipPasswordMode.FIELD: value = values.get(field_name) password = "" if value is None else str(value) if not password: qualifier = "campaign-global" if scope == ZipPasswordScope.GLOBAL.value else "recipient" raise ZipBuildError( "zip_password_value_missing", f"The {qualifier} ZIP password field {field_name!r} is empty for archive {archive.name!r}", ) return password def _archive_filename(archive: ZipArchiveConfig, values: dict[str, Any], entry_index: int) -> str: rendered = _render_template(archive.name or "attachments.zip", values, keep_missing=False) filename = _safe_filename(rendered, f"entry-{entry_index:04d}-attachments.zip") return filename if filename.lower().endswith(".zip") else f"{filename}.zip" def _unique_attachment_filename(filename: str, used: set[str]) -> str: candidate = filename path = Path(filename) counter = 2 while candidate.casefold() in used: candidate = f"{path.stem} ({counter}){path.suffix}" counter += 1 used.add(candidate.casefold()) return candidate def _deduplicated_archive_members(members: list[tuple[Path, str]]) -> list[tuple[Path, str]]: unique: list[tuple[Path, str]] = [] seen: set[str] = set() for path, archive_name in members: key = str(path.resolve()) if key in seen: continue seen.add(key) unique.append((path, archive_name)) return unique def _attachment_filename_values( *, values: dict[str, Any], attachment: ResolvedAttachment, path: Path, position: int, ) -> dict[str, Any]: filename_values = dict(values) filename_values.update({ "file.name": path.name, "file.stem": path.stem, "file.suffix": path.suffix, "file.ext": path.suffix[1:] if path.suffix.startswith(".") else path.suffix, "file.index": position, "attachment.id": attachment.attachment_id or "", "attachment.label": attachment.label or "", }) return filename_values def _render_attachment_filename( *, template: str | None, attachment: ResolvedAttachment, path: Path, values: dict[str, Any], position: int, ) -> str: if not template: return path.name rendered = _render_template( template, _attachment_filename_values(values=values, attachment=attachment, path=path, position=position), keep_missing=False, ) filename = _safe_filename(rendered, path.name) if path.suffix and not Path(filename).suffix: filename = f"{filename}{path.suffix}" return filename def _attach_files( *, message: EmailMessage, config: CampaignConfig, entry: EntryConfig, entry_index: int, resolution: EntryAttachmentResolution, values: dict[str, Any], work_dir: Path, ) -> int: attached_count = 0 archive_members: dict[str, list[tuple[Path, str]]] = {} archive_attachments: dict[str, list[ResolvedAttachment]] = {} used_message_filenames: set[str] = set() used_zip_member_filenames: dict[str, set[str]] = {} for attachment in resolution.attachments: attachment.message_filenames = [] attachment.zip_entry_names = [] for attachment in resolution.attachments: if attachment.status != AttachmentMatchStatus.OK or not attachment.matches: continue match_paths = [Path(match) for match in attachment.matches] if attachment.zip_enabled and attachment.zip_archive_id: used_archive_names = used_zip_member_filenames.setdefault(attachment.zip_archive_id, set()) for position, path in enumerate(match_paths, start=1): requested = _render_attachment_filename( template=attachment.zip_entry_name_template, attachment=attachment, path=path, values=values, position=position, ) archive_name = _unique_attachment_filename(requested, used_archive_names) archive_members.setdefault(attachment.zip_archive_id, []).append((path, archive_name)) attachment.zip_entry_names.append(archive_name) archive_attachments.setdefault(attachment.zip_archive_id, []).append(attachment) continue for position, path in enumerate(match_paths, start=1): requested = _render_attachment_filename( template=attachment.message_filename_template, attachment=attachment, path=path, values=values, position=position, ) filename = _unique_attachment_filename(requested, used_message_filenames) attachment.message_filenames.append(filename) data, maintype, subtype = _attachment_bytes(path) message.add_attachment(data, maintype=maintype, subtype=subtype, filename=filename) attached_count += 1 for archive in config.attachments.zip.archives: members = _deduplicated_archive_members(archive_members.get(archive.id, [])) if not members: continue filename = _unique_attachment_filename(_archive_filename(archive, values, entry_index), used_message_filenames) password = _zip_password(archive, values) archive_path = create_zip_archive( work_dir / "_zip" / f"entry-{entry_index:04d}" / _safe_filename(archive.id, "archive") / filename, members, password, ) data, maintype, subtype = _attachment_bytes(archive_path) message.add_attachment(data, maintype=maintype, subtype=subtype, filename=filename) attached_count += 1 for attachment in archive_attachments.get(archive.id, []): attachment.zip_filename = filename return attached_count def _imap_initial_status(config: CampaignConfig) -> ImapStatus: if config.delivery.imap_append_sent.enabled: return ImapStatus.PENDING return ImapStatus.NOT_REQUESTED def _write_eml(message: EmailMessage, output_dir: Path, entry: EntryConfig, entry_index: int) -> tuple[str, int]: output_dir.mkdir(parents=True, exist_ok=True) filename = _safe_filename(entry.id, f"entry-{entry_index:04d}") + ".eml" path = output_dir / filename path.write_bytes(bytes(message)) return str(path), path.stat().st_size def build_entry_message( *, config: CampaignConfig, campaign_file: str | Path, entry: EntryConfig, entry_index: int, output_dir: Path | None = None, write_eml: bool = False, work_dir: Path | None = None, attachment_match_index: AttachmentMatchIndex | None = None, ) -> BuiltMessage: resolution = resolve_entry_attachments(config=config, campaign_file=campaign_file, entry=entry, entry_index=entry_index, match_index=attachment_match_index) effective_addresses = effective_address_lists(config, entry) senders = effective_addresses["from"] sender = senders[0] if senders else None recipients = {key: value for key, value in effective_addresses.items() if key != "from"} issues = _message_issues_from_attachment_resolution(resolution) validation_status = _validation_status_from_attachment_status(resolution.status) ignored_field_overrides = ignored_entry_field_overrides(config, entry) if ignored_field_overrides: issues.append( MessageIssue( severity="warning", code="field_override_not_allowed", message="Recipient field value(s) ignored because the campaign field does not allow overrides: " + ", ".join(ignored_field_overrides), behavior="warn", source="fields", ) ) if validation_status == MessageValidationStatus.READY: validation_status = MessageValidationStatus.WARNING if not entry.active: draft = MessageDraft( entry_index=entry_index, entry_id=entry.id, active=False, build_status=BuildStatus.BUILD_FAILED, validation_status=MessageValidationStatus.INACTIVE, send_status=SendStatus.DRAFT, imap_status=ImapStatus.SKIPPED, from_=_message_address(sender), from_all=_message_addresses(senders), to=_message_addresses(recipients["to"]), cc=_message_addresses(recipients["cc"]), bcc=_message_addresses(recipients["bcc"]), reply_to=_message_addresses(recipients["reply_to"]), bounce_to=_message_addresses(recipients["bounce_to"]), disposition_notification_to=_message_addresses(recipients["disposition_notification_to"]), attachments=_attachment_summaries(resolution), issues=[MessageIssue(severity="info", code="inactive_entry", message="Entry is inactive", behavior=config.validation_policy.inactive_entry.value, source="entry")], ) return BuiltMessage(draft=draft, mime=None) if not recipients["to"]: behavior = config.validation_policy.missing_email.value issues.append(_issue_from_behavior(code="missing_email", message="No effective To recipient is configured", behavior=behavior, source="recipients")) validation_status = MessageValidationStatus.BLOCKED if behavior == MissingAddressBehavior.BLOCK.value else MessageValidationStatus.EXCLUDED subject_template, text_template, html_template = _load_template_parts(config, campaign_file) body_mode = _template_body_mode(config) values = build_template_values(config, entry) keep_missing_placeholders = not config.validation_policy.ignore_empty_fields subject = _render_template(subject_template, values, keep_missing=keep_missing_placeholders) text_body = _render_template(text_template or "", values, keep_missing=keep_missing_placeholders) if text_template is not None else None html_body = _render_template(html_template or "", values, keep_missing=keep_missing_placeholders) if html_template is not None else None unresolved_fields = _find_unresolved_placeholders(subject) if body_mode in {TemplateBodyMode.TEXT.value, TemplateBodyMode.BOTH.value}: unresolved_fields |= _find_unresolved_placeholders(text_body) if body_mode in {TemplateBodyMode.HTML.value, TemplateBodyMode.BOTH.value}: unresolved_fields |= _find_unresolved_placeholders(html_body) unresolved = sorted(unresolved_fields) if unresolved: behavior = config.validation_policy.template_error.value issues.append( _issue_from_behavior( code="template_error", message="Unresolved template placeholder(s): " + ", ".join(unresolved), behavior=behavior, source="template", ) ) validation_status = MessageValidationStatus.BLOCKED if behavior == MissingAddressBehavior.BLOCK.value else MessageValidationStatus.EXCLUDED message = EmailMessage() try: message["Date"] = formatdate(localtime=True) message["Message-ID"] = make_msgid() if senders: message["From"] = _format_recipient_header(senders) if len(senders) > 1: # RFC 5322 requires a singular Sender when From contains more # than one mailbox. The first effective From address remains # the SMTP envelope sender and compatibility primary value. message["Sender"] = _format_recipient(senders[0]) if recipients["to"]: message["To"] = _format_recipient_header(recipients["to"]) if recipients["cc"]: message["Cc"] = _format_recipient_header(recipients["cc"]) # Bcc deliberately remains envelope-only and is tracked in MessageDraft. if recipients["reply_to"]: message["Reply-To"] = _format_recipient_header(recipients["reply_to"]) if recipients["disposition_notification_to"]: message["Disposition-Notification-To"] = _format_recipient_header(recipients["disposition_notification_to"]) # bounce_to is tracked but not emitted as Return-Path. That should be the SMTP envelope sender. message["Subject"] = subject effective_html_body = html_body if html_body and html_body.strip() else None if body_mode == TemplateBodyMode.HTML.value: message.set_content(effective_html_body or "", subtype="html") elif body_mode == TemplateBodyMode.TEXT.value: message.set_content(text_body or "") elif effective_html_body is not None: message.set_content(text_body or "") message.add_alternative(effective_html_body, subtype="html") else: message.set_content(text_body or "") if work_dir is None: work_dir = output_dir or Path(tempfile.mkdtemp(prefix="multimailer-build-")) attachment_count = _attach_files( message=message, config=config, entry=entry, entry_index=entry_index, resolution=resolution, values=values, work_dir=work_dir, ) build_status = BuildStatus.BUILT except ZipBuildError as exc: issues.append(MessageIssue(severity="error", code=exc.code, message=str(exc), behavior="block", source="attachments")) validation_status = MessageValidationStatus.BLOCKED build_status = BuildStatus.BUILD_FAILED attachment_count = 0 message = None # type: ignore[assignment] except Exception as exc: issues.append(MessageIssue(severity="error", code="build_failed", message=str(exc), behavior="block", source="builder")) validation_status = MessageValidationStatus.BLOCKED build_status = BuildStatus.BUILD_FAILED attachment_count = 0 message = None # type: ignore[assignment] eml_path: str | None = None eml_size: int | None = None if write_eml and output_dir is not None and message is not None: eml_path, eml_size = _write_eml(message, output_dir, entry, entry_index) draft = MessageDraft( entry_index=entry_index, entry_id=entry.id, active=entry.active, build_status=build_status, validation_status=validation_status, send_status=SendStatus.DRAFT, imap_status=_imap_initial_status(config) if build_status == BuildStatus.BUILT else ImapStatus.SKIPPED, subject=subject, from_=_message_address(sender), from_all=_message_addresses(senders), to=_message_addresses(recipients["to"]), cc=_message_addresses(recipients["cc"]), bcc=_message_addresses(recipients["bcc"]), reply_to=_message_addresses(recipients["reply_to"]), bounce_to=_message_addresses(recipients["bounce_to"]), disposition_notification_to=_message_addresses(recipients["disposition_notification_to"]), attachment_count=attachment_count, attachments=_attachment_summaries(resolution), issues=issues, eml_path=eml_path, eml_size_bytes=eml_size, ) return BuiltMessage(draft=draft, mime=message) def _unsent_attachment_issues( *, config: CampaignConfig, campaign_file: str | Path, built_messages: list[BuiltMessage], attachment_match_index: AttachmentMatchIndex | None = None, ) -> list[MessageIssue]: behavior = config.validation_policy.unsent_attachment_files.value if behavior == Behavior.CONTINUE.value: return [] matched_files = { Path(match).resolve() for built in built_messages for attachment in built.draft.attachments for match in attachment.matches } issues: list[MessageIssue] = [] for base_path in config.attachments.base_paths: if not base_path.unsent_warning: continue directory = _resolve(campaign_file, base_path.path) if not directory.exists() or not directory.is_dir(): continue if attachment_match_index is not None: all_files = sorted(path.resolve() for path in attachment_match_index.iter_files(directory, recursive=True)) else: all_files = sorted(path.resolve() for path in directory.rglob("*") if path.is_file()) unsent = [path for path in all_files if path not in matched_files] if not unsent: continue shown = ", ".join(str(path.relative_to(directory)) for path in unsent[:10]) if len(unsent) > 10: shown += f", … (+{len(unsent) - 10} more)" issues.append( _issue_from_behavior( code="unsent_attachment_files", message=f"{len(unsent)} file(s) in attachment source {base_path.name!r} are not used by any message: {shown}", behavior=behavior, source=f"attachments:{base_path.name}", ) ) return issues def _apply_campaign_level_issues(built_messages: list[BuiltMessage], issues: list[MessageIssue]) -> None: if not issues: return for built in built_messages: if not built.draft.active: continue built.draft.issues.extend(issues) status = built.draft.validation_status for issue in issues: if issue.behavior: status = _apply_behavior(status, issue.behavior) built.draft.validation_status = status def build_campaign_messages( config: CampaignConfig, *, campaign_file: str | Path, output_dir: str | Path | None = None, write_eml: bool = False, ) -> CampaignBuildResult: campaign_path = Path(campaign_file).resolve() entries = load_campaign_entries(config, campaign_file=campaign_path) output_path = Path(output_dir).resolve() if output_dir is not None else None started = time.perf_counter() attachment_match_index = AttachmentMatchIndex() with tempfile.TemporaryDirectory(prefix="multimailer-build-") as tmp: work_dir = output_path or Path(tmp) built_messages = [ build_entry_message( config=config, campaign_file=campaign_path, entry=entry, entry_index=index, output_dir=output_path, write_eml=write_eml, work_dir=work_dir, attachment_match_index=attachment_match_index, ) for index, entry in enumerate(entries, start=1) if entry.active ] _apply_campaign_level_issues( built_messages, _unsent_attachment_issues( config=config, campaign_file=campaign_path, built_messages=built_messages, attachment_match_index=attachment_match_index, ), ) rules_resolved = sum(len(built.draft.attachments) for built in built_messages) report = CampaignBuildReport( campaign_id=config.campaign.id, campaign_name=config.campaign.name, campaign_file=str(campaign_path), entries_count=len(entries), inactive_entries_count=sum(1 for entry in entries if not entry.active), messages=[built.draft for built in built_messages], attachment_resolution_profile=attachment_match_index.stats( duration_ms=(time.perf_counter() - started) * 1000, rules_resolved=rules_resolved, ), ) return CampaignBuildResult(report=report, built_messages=built_messages)