from __future__ import annotations import hashlib import json from email import policy from email.parser import BytesParser from pathlib import Path from typing import Any import copy from datetime import UTC, datetime from uuid import uuid4 from sqlalchemy import func from sqlalchemy.orm import Session from govoplan_campaign.backend.db.models import ( Campaign, CampaignIssue, CampaignJob, CampaignStatus, CampaignVersion, CampaignVersionWorkflowState, JobImapStatus, JobQueueStatus, JobSendStatus, JobValidationStatus, ) from govoplan_campaign.backend.campaign.loader import load_campaign_config, load_campaign_json, validate_against_schema from govoplan_campaign.backend.campaign.validation import validate_campaign_config from govoplan_campaign.backend.messages.builder import build_campaign_messages from govoplan_campaign.backend.messages.models import MessageDraft from govoplan_campaign.backend.sending.execution import create_execution_snapshot from govoplan_campaign.backend.campaign.models import CampaignConfig from govoplan_campaign.backend.integrations import files_integration, mail_integration RUNTIME_DIR = Path(__file__).resolve().parents[3] / "runtime" CAMPAIGN_SNAPSHOT_DIR = RUNTIME_DIR / "campaign_snapshots" BUILD_OUTPUT_DIR = RUNTIME_DIR / "generated_eml" class CampaignPersistenceError(RuntimeError): pass def _ensure_dirs() -> None: CAMPAIGN_SNAPSHOT_DIR.mkdir(parents=True, exist_ok=True) BUILD_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) def load_campaign_config_from_json( session: Session, *, tenant_id: str, raw_json: dict[str, Any], campaign_id: str | None = None, owner_user_id: str | None = None, owner_group_id: str | None = None, ) -> CampaignConfig: materialized = mail_integration().materialize_campaign_mail_profile_config( session, tenant_id=tenant_id, raw_json=raw_json, campaign_id=campaign_id, owner_user_id=owner_user_id, owner_group_id=owner_group_id, ) validate_against_schema(materialized) return CampaignConfig.model_validate(materialized) def _write_campaign_snapshot(version: CampaignVersion) -> Path: _ensure_dirs() path = CAMPAIGN_SNAPSHOT_DIR / f"{version.id}.json" path.write_text(json.dumps(version.raw_json, ensure_ascii=False, indent=2), encoding="utf-8") return path def _next_version_number(session: Session, campaign_id: str) -> int: current = session.query(func.max(CampaignVersion.version_number)).filter(CampaignVersion.campaign_id == campaign_id).scalar() return int(current or 0) + 1 def _resolve_runtime_path(base_path: Path | None, value: str | None) -> str | None: if not value or base_path is None: return value path = Path(value).expanduser() if path.is_absolute(): return str(path) return str((base_path / path).resolve()) def normalize_campaign_paths(raw_json: dict[str, Any], source_base_path: str | Path | None) -> dict[str, Any]: """Return a DB/runtime-safe campaign JSON snapshot. The CLI naturally resolves relative paths against the campaign.json file. Once the campaign is stored in the database, the JSON snapshot lives in app/mailer/runtime/campaign_snapshots. To keep existing file-based campaigns working, relative file paths are normalized to absolute paths at import time when a source_base_path is known. """ base = Path(source_base_path).expanduser().resolve() if source_base_path else None data = copy.deepcopy(raw_json) template_source = data.get("template", {}).get("source") if isinstance(data.get("template"), dict) else None if isinstance(template_source, dict): for key in ("subject_path", "text_path", "html_path"): template_source[key] = _resolve_runtime_path(base, template_source.get(key)) entries_source = data.get("entries", {}).get("source") if isinstance(data.get("entries"), dict) else None if isinstance(entries_source, dict): entries_source["path"] = _resolve_runtime_path(base, entries_source.get("path")) attachments = data.get("attachments") if isinstance(attachments, dict): attachments["base_path"] = _resolve_runtime_path(base, attachments.get("base_path")) or "." return data def create_campaign_version_from_json( session: Session, *, tenant_id: str, user_id: str | None, raw_json: dict[str, Any], source_filename: str | None = None, source_base_path: str | None = None, ) -> tuple[Campaign, CampaignVersion]: if source_base_path is None and source_filename: source_path = Path(source_filename).expanduser() source_base_path = str(source_path.parent if source_path.suffix else source_path) runtime_json = normalize_campaign_paths(raw_json, source_base_path) config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=runtime_json, owner_user_id=user_id) campaign = ( session.query(Campaign) .filter(Campaign.tenant_id == tenant_id, Campaign.external_id == config.campaign.id) .one_or_none() ) if campaign is None: campaign = Campaign( tenant_id=tenant_id, created_by_user_id=user_id, owner_user_id=user_id, external_id=config.campaign.id, name=config.campaign.name, description=config.campaign.description, status=CampaignStatus.DRAFT.value, ) session.add(campaign) session.flush() else: current = session.get(CampaignVersion, campaign.current_version_id) if campaign.current_version_id else None if current and not _version_is_audit_safe_snapshot(current): raise CampaignPersistenceError( f"Campaign already has active working version #{current.version_number}. " "Continue editing or unlock that version instead of importing a parallel draft." ) campaign.name = config.campaign.name campaign.description = config.campaign.description version = CampaignVersion( campaign_id=campaign.id, version_number=_next_version_number(session, campaign.id), raw_json=runtime_json, schema_version=raw_json.get("version", "1.0"), source_filename=source_filename, source_base_path=source_base_path, ) session.add(version) session.flush() campaign.current_version_id = version.id session.add(campaign) _write_campaign_snapshot(version) session.commit() return campaign, version def _version_user_lock_state(version: CampaignVersion) -> str | None: state = getattr(version, "user_lock_state", None) if state in {"temporary", "permanent"}: return state return "permanent" if version.published_at else None def _version_is_user_locked(version: CampaignVersion) -> bool: return _version_user_lock_state(version) is not None def _version_is_audit_safe_snapshot(version: CampaignVersion) -> bool: return _version_user_lock_state(version) == "permanent" or version.workflow_state in { CampaignVersionWorkflowState.QUEUED.value, CampaignVersionWorkflowState.SENDING.value, CampaignVersionWorkflowState.COMPLETED.value, CampaignVersionWorkflowState.PARTIALLY_COMPLETED.value, CampaignVersionWorkflowState.OUTCOME_UNKNOWN.value, CampaignVersionWorkflowState.FAILED.value, CampaignVersionWorkflowState.CANCELLED.value, CampaignVersionWorkflowState.ARCHIVED.value, } def _ensure_current_campaign_version(campaign: Campaign, version: CampaignVersion, *, action: str) -> None: if campaign.current_version_id != version.id: raise CampaignPersistenceError( f"Historical campaign versions are read-only and cannot be used to {action}. " "Open the current working version instead." ) def _version_is_validated_and_locked(version: CampaignVersion) -> bool: validation_summary = version.validation_summary if isinstance(version.validation_summary, dict) else {} return bool(version.locked_at and validation_summary.get("ok") is True and not _version_is_user_locked(version)) def _ensure_version_validated_and_locked(version: CampaignVersion) -> None: state = _version_user_lock_state(version) if state == "temporary": raise CampaignPersistenceError("This version has a temporary user lock. Unlock it before building, queueing, dry-run or sending.") if state == "permanent": raise CampaignPersistenceError("This version is permanently user-locked. Create an editable copy instead.") if not _version_is_validated_and_locked(version): raise CampaignPersistenceError("Campaign version must be validated and locked before building, queueing, dry-run or sending.") def load_version_config(session: Session, version_id: str): version = session.get(CampaignVersion, version_id) if not version: raise CampaignPersistenceError(f"Campaign version not found: {version_id}") campaign = session.get(Campaign, version.campaign_id) if not campaign: raise CampaignPersistenceError(f"Campaign not found for version: {version_id}") path = _write_campaign_snapshot(version) raw_json = version.raw_json if isinstance(version.raw_json, dict) else {} return version, path, load_campaign_config_from_json(session, tenant_id=campaign.tenant_id, raw_json=raw_json, campaign_id=campaign.id) def validate_campaign_version( session: Session, *, tenant_id: str, version_id: str, check_files: bool = False, user_id: str | None = None, lock_on_success: bool = True, ) -> dict[str, Any]: version, snapshot_path, config = load_version_config(session, version_id) campaign = session.get(Campaign, version.campaign_id) if not campaign or campaign.tenant_id != tenant_id: raise CampaignPersistenceError("Campaign version is not accessible for this tenant") _ensure_current_campaign_version(campaign, version, action="validate") if _version_is_user_locked(version) or version.workflow_state in { CampaignVersionWorkflowState.QUEUED.value, CampaignVersionWorkflowState.SENDING.value, CampaignVersionWorkflowState.COMPLETED.value, CampaignVersionWorkflowState.PARTIALLY_COMPLETED.value, CampaignVersionWorkflowState.OUTCOME_UNKNOWN.value, CampaignVersionWorkflowState.FAILED.value, CampaignVersionWorkflowState.CANCELLED.value, CampaignVersionWorkflowState.ARCHIVED.value, }: lock_label = "temporarily user-locked" if _version_user_lock_state(version) == "temporary" else "permanently locked/final" raise CampaignPersistenceError(f"{lock_label.capitalize()} campaign versions cannot be validated. Unlock or create an editable copy instead.") if check_files: files = files_integration() with files.prepared_campaign_snapshot( session, tenant_id=tenant_id, campaign_id=campaign.id, raw_json=version.raw_json if isinstance(version.raw_json, dict) else {}, include_bytes=False, prefix="govoplan-managed-validate-", ) as prepared: managed_raw = load_campaign_json(prepared.path) managed_config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=managed_raw, campaign_id=campaign.id) report = validate_campaign_config(managed_config, campaign_file=prepared.path, check_files=True) else: report = validate_campaign_config(config, campaign_file=snapshot_path, check_files=False) report_json = report.model_dump(mode="json") report_json.update({"ok": report.ok, "error_count": report.error_count, "warning_count": report.warning_count}) version.validation_summary = report_json # Replace version-level semantic issues from previous validations. ( session.query(CampaignIssue) .filter(CampaignIssue.campaign_version_id == version.id, CampaignIssue.job_id.is_(None)) .delete(synchronize_session=False) ) for issue in report.issues: session.add( CampaignIssue( tenant_id=tenant_id, campaign_id=campaign.id, campaign_version_id=version.id, severity=issue.severity.value, code=issue.code, message=issue.message, source=issue.path, ) ) campaign.status = CampaignStatus.VALIDATED.value if report.ok else CampaignStatus.NEEDS_REVIEW.value if report.ok: version.workflow_state = CampaignVersionWorkflowState.APPROVED.value version.is_complete = True if lock_on_success and version.locked_at is None: from datetime import UTC, datetime version.locked_at = datetime.now(UTC) version.locked_by_user_id = user_id else: version.workflow_state = CampaignVersionWorkflowState.EDITING.value session.add(version) session.add(campaign) session.commit() return report_json def _job_validation_status(value: str) -> str: allowed = {item.value for item in JobValidationStatus} return value if value in allowed else JobValidationStatus.NEEDS_REVIEW.value def _eml_evidence(eml_path: str | None) -> tuple[str | None, str | None]: if not eml_path: return None, None path = Path(eml_path) if not path.exists(): return None, None payload = path.read_bytes() message_id = BytesParser(policy=policy.default).parsebytes(payload).get("Message-ID") return hashlib.sha256(payload).hexdigest(), str(message_id) if message_id else None def _job_from_message( *, tenant_id: str, campaign_id: str, version_id: str, message: MessageDraft, ) -> CampaignJob: recipient_email = message.to[0].email if message.to else None eml_sha256, message_id_header = _eml_evidence(message.eml_path) return CampaignJob( tenant_id=tenant_id, campaign_id=campaign_id, campaign_version_id=version_id, entry_index=message.entry_index, entry_id=message.entry_id, recipient_email=recipient_email, subject=message.subject, message_id_header=message_id_header, eml_local_path=message.eml_path, eml_size_bytes=message.eml_size_bytes, eml_sha256=eml_sha256, build_status=message.build_status.value if hasattr(message.build_status, "value") else str(message.build_status), validation_status=_job_validation_status(message.validation_status.value), queue_status=JobQueueStatus.DRAFT.value, send_status=JobSendStatus.NOT_QUEUED.value, imap_status=message.imap_status.value if hasattr(message.imap_status, "value") else JobImapStatus.NOT_REQUESTED.value, resolved_recipients={ "from": message.from_.model_dump(mode="json") if message.from_ else None, "from_all": [item.model_dump(mode="json") for item in message.from_all], "to": [item.model_dump(mode="json") for item in message.to], "cc": [item.model_dump(mode="json") for item in message.cc], "bcc": [item.model_dump(mode="json") for item in message.bcc], "reply_to": [item.model_dump(mode="json") for item in message.reply_to], "bounce_to": [item.model_dump(mode="json") for item in message.bounce_to], "disposition_notification_to": [item.model_dump(mode="json") for item in message.disposition_notification_to], }, resolved_attachments=[files_integration().public_attachment_summary_payload(item) for item in message.attachments], issues_snapshot=[item.model_dump(mode="json") for item in message.issues], last_error="; ".join(issue.message for issue in message.issues if issue.severity == "error") or None, ) def build_campaign_version( session: Session, *, tenant_id: str, version_id: str, write_eml: bool = True, ) -> dict[str, Any]: version, snapshot_path, config = load_version_config(session, version_id) campaign = session.get(Campaign, version.campaign_id) if not campaign or campaign.tenant_id != tenant_id: raise CampaignPersistenceError("Campaign version is not accessible for this tenant") _ensure_current_campaign_version(campaign, version, action="build") if version.workflow_state == CampaignVersionWorkflowState.COMPLETED.value: raise CampaignPersistenceError("Sent campaign versions cannot be rebuilt") validation_summary = version.validation_summary if isinstance(version.validation_summary, dict) else {} if not validation_summary.get("ok"): raise CampaignPersistenceError("Campaign version must be successfully validated before messages are built") _ensure_version_validated_and_locked(version) output_dir = BUILD_OUTPUT_DIR / campaign.id / version.id files = files_integration() with files.prepared_campaign_snapshot( session, tenant_id=tenant_id, campaign_id=campaign.id, raw_json=version.raw_json if isinstance(version.raw_json, dict) else {}, include_bytes=True, prefix="govoplan-managed-build-", ) as prepared: managed_raw = load_campaign_json(prepared.path) managed_config = load_campaign_config_from_json(session, tenant_id=tenant_id, raw_json=managed_raw, campaign_id=campaign.id) result = build_campaign_messages(managed_config, campaign_file=prepared.path, output_dir=output_dir, write_eml=write_eml) files.annotate_built_messages_with_managed_files(result.built_messages, prepared.managed_files_by_local_path) report_json = result.report.model_dump(mode="json", by_alias=True) for message_payload, message in zip(report_json.get("messages", []), result.report.messages, strict=False): if isinstance(message_payload, dict): message_payload["attachments"] = [files.public_attachment_summary_payload(item) for item in message.attachments] report_json["built_at"] = datetime.now(UTC).isoformat() report_json["build_token"] = uuid4().hex report_json.update({ "built_count": result.report.built_count, "build_failed_count": result.report.build_failed_count, "ready_count": result.report.ready_count, "warning_count": result.report.warning_count, "needs_review_count": result.report.needs_review_count, "blocked_count": result.report.blocked_count, "excluded_count": result.report.excluded_count, "inactive_count": result.report.inactive_count, "queueable_count": result.report.queueable_count, }) version.build_summary = report_json editor_state = copy.deepcopy(version.editor_state or {}) editor_state.pop("review_send", None) version.editor_state = editor_state # Rebuild jobs for the current version. Later, protect sent jobs from destructive rebuilds. session.query(CampaignIssue).filter(CampaignIssue.campaign_version_id == version.id, CampaignIssue.job_id.is_not(None)).delete(synchronize_session=False) session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version.id).delete(synchronize_session=False) session.flush() job_build_pairs: list[tuple[CampaignJob, MessageDraft]] = [] for built in result.built_messages: job = _job_from_message( tenant_id=tenant_id, campaign_id=campaign.id, version_id=version.id, message=built.draft, ) session.add(job) job_build_pairs.append((job, built.draft)) # Assign all job IDs in one round-trip, then persist exact attachment use # records in bulk. This avoids one flush plus several metadata queries per # recipient for large campaigns. session.flush() files.record_campaign_attachment_uses_for_jobs( session, [job for job, _message in job_build_pairs], stage="built", ) runtime_smtp = managed_config.server.runtime_smtp_config() if not runtime_smtp: raise CampaignPersistenceError("Campaign has no SMTP configuration; an execution snapshot cannot be created") execution_snapshot, execution_snapshot_hash = create_execution_snapshot( version, smtp=runtime_smtp, imap=managed_config.server.runtime_imap_config(), delivery=managed_config.delivery, jobs=[job for job, _message in job_build_pairs], build_summary=report_json, ) version.execution_snapshot = execution_snapshot version.execution_snapshot_hash = execution_snapshot_hash version.execution_snapshot_at = datetime.now(UTC) for job, message in job_build_pairs: for issue in message.issues: session.add( CampaignIssue( tenant_id=tenant_id, campaign_id=campaign.id, campaign_version_id=version.id, job_id=job.id, severity=issue.severity, code=issue.code, message=issue.message, source=issue.source, behavior=issue.behavior, ) ) if result.report.needs_review_count or result.report.blocked_count: campaign.status = CampaignStatus.NEEDS_REVIEW.value version.workflow_state = CampaignVersionWorkflowState.APPROVED.value elif result.report.queueable_count > 0: campaign.status = CampaignStatus.READY_TO_QUEUE.value version.workflow_state = CampaignVersionWorkflowState.BUILT.value else: campaign.status = CampaignStatus.VALIDATED.value session.add(version) session.add(campaign) session.commit() return report_json